From ce1782b966e9c74e25905d5f28563851a9dffb5c Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 5 Sep 2024 11:32:12 -0700 Subject: [PATCH 01/32] check in --- ...olarisEclipseLinkMetaStoreSessionImpl.java | 9 +- .../polaris/core/catalog/PaginationToken.java | 86 +++++++++++++++++++ .../PolarisMetaStoreManagerImpl.java | 8 +- .../persistence/PolarisMetaStoreSession.java | 6 +- .../PolarisTreeMapMetaStoreSessionImpl.java | 9 +- 5 files changed, 107 insertions(+), 11 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/catalog/PaginationToken.java diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index dec10d3a4..ecc9d45e1 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -47,6 +47,8 @@ import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; + +import org.apache.polaris.core.catalog.PaginationToken; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -522,7 +524,7 @@ public List lookupEntityActiveBatch( catalogId, parentId, entityType, - Integer.MAX_VALUE, + PaginationToken.readEverything(), entityFilter, entity -> new PolarisEntityActiveRecord( @@ -540,7 +542,7 @@ public List lookupEntityActiveBatch( long catalogId, long parentId, @NotNull PolarisEntityType entityType, - int limit, + @NotNull PaginationToken paginationToken, @NotNull Predicate entityFilter, @NotNull Function transformer) { // full range scan under the parent for that type @@ -549,7 +551,8 @@ public List lookupEntityActiveBatch( .stream() .map(ModelEntity::toEntity) .filter(entityFilter) - .limit(limit) +// .skip(paginationToken.offset()) +// .limit(paginationToken.pageSize()) .map(transformer) .collect(Collectors.toList()); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PaginationToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PaginationToken.java new file mode 100644 index 000000000..cfdfe75a9 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PaginationToken.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.catalog; + +import java.util.Base64; + +public class PaginationToken { + + public final long offset; + public final long pageSize; + + public static final String TOKEN_PREFIX = "polaris"; + public static final long TOKEN_START = 0; + public static final long DEFAULT_PAGE_SIZE = 1000; + + public PaginationToken(long offset, long pageSize) { + this.offset = offset; + this.pageSize = pageSize; + } + + /** + * Construct a PaginationToken from a plain limit + */ + public static PaginationToken fromLimit(int limit) { + return new PaginationToken(TOKEN_START, limit); + } + + /** + * Construct a PaginationToken from a plain limit + */ + public static PaginationToken readEverything() { + return new PaginationToken(TOKEN_START, Integer.MAX_VALUE); + } + + /** + * Decode a token string into a PaginationToken object + */ + public static PaginationToken fromString(String tokenString) { + if (tokenString == null || tokenString.isEmpty()) { + return new PaginationToken(TOKEN_START, DEFAULT_PAGE_SIZE); + } + + if (!tokenString.startsWith(TOKEN_PREFIX)) { + throw new IllegalArgumentException("Invalid token format"); + } + + try { + String decoded = new String(Base64.getDecoder().decode(tokenString)); + String[] parts = decoded.split(":"); + + if (parts.length != 3 || !parts[0].equals(TOKEN_PREFIX)) { + throw new IllegalArgumentException("Invalid token format"); + } + + long offset = Long.parseLong(parts[1]); + long pageSize = Long.parseLong(parts[2]); + + return new PaginationToken(offset, pageSize); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to decode token: " + tokenString, e); + } + } + + @Override + public String toString() { + String tokenContent = TOKEN_PREFIX + ":" + offset + ":" + pageSize; + return Base64.getEncoder().encodeToString(tokenContent.getBytes()); + } +} \ No newline at end of file diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index b05129fa3..560b1d3a2 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -33,6 +33,8 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; + +import org.apache.polaris.core.catalog.PaginationToken; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -1481,7 +1483,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str catalogId, catalogId, PolarisEntityType.CATALOG_ROLE, - 2, + PaginationToken.fromLimit(2), entity -> true, Function.identity()); @@ -2013,7 +2015,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @NotNull PolarisCallContext callCtx, @NotNull PolarisMetaStoreSession ms, String executorId, - int limit) { + PaginationToken paginationToken) { // find all available tasks List availableTasks = @@ -2022,7 +2024,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( PolarisEntityConstants.getRootEntityId(), PolarisEntityConstants.getRootEntityId(), PolarisEntityType.TASK, - limit, + paginationToken, entity -> { PolarisObjectMapperUtil.TaskExecutionState taskState = PolarisObjectMapperUtil.parseTaskState(entity); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index ff5bcbf3e..23899a3b1 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -22,6 +22,8 @@ import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; + +import org.apache.polaris.core.catalog.PaginationToken; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; @@ -339,7 +341,7 @@ List listActiveEntities( * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level * @param parentId id of the parent, can be the special 0 value representing the root entity * @param entityType type of entities to list - * @param limit the max number of items to return + * @param paginationToken the pagination token to use * @param entityFilter the filter to be applied to each entity. Only entities where the predicate * returns true are returned in the list * @param transformer the transformation function applied to the {@link PolarisBaseEntity} before @@ -352,7 +354,7 @@ List listActiveEntities( long catalogId, long parentId, @NotNull PolarisEntityType entityType, - int limit, + @NotNull PaginationToken paginationToken, @NotNull Predicate entityFilter, @NotNull Function transformer); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java index 096bce49e..939f12397 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java @@ -24,6 +24,8 @@ import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; + +import org.apache.polaris.core.catalog.PaginationToken; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; @@ -340,7 +342,7 @@ public List lookupEntityActiveBatch( catalogId, parentId, entityType, - Integer.MAX_VALUE, + PaginationToken.readEverything(), entityFilter, entity -> new PolarisEntityActiveRecord( @@ -358,7 +360,7 @@ public List lookupEntityActiveBatch( long catalogId, long parentId, @NotNull PolarisEntityType entityType, - int limit, + PaginationToken paginationToken, @NotNull Predicate entityFilter, @NotNull Function transformer) { // full range scan under the parent for that type @@ -367,7 +369,8 @@ public List lookupEntityActiveBatch( .readRange(this.store.buildPrefixKeyComposite(catalogId, parentId, entityType.getCode())) .stream() .filter(entityFilter) - .limit(limit) + .skip(paginationToken.offset) + .limit(paginationToken.pageSize) .map(transformer) .collect(Collectors.toList()); } From 63ef91652ec920c9f73f4d18e061cad8ba717018 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 5 Sep 2024 13:59:58 -0700 Subject: [PATCH 02/32] refactor --- ...PolarisEclipseLinkMetaStoreSessionImpl.java | 6 +++--- .../{PaginationToken.java => PageToken.java} | 18 +++++++++--------- .../PolarisMetaStoreManagerImpl.java | 8 ++++---- .../persistence/PolarisMetaStoreSession.java | 6 +++--- .../PolarisTreeMapMetaStoreSessionImpl.java | 10 +++++----- 5 files changed, 24 insertions(+), 24 deletions(-) rename polaris-core/src/main/java/org/apache/polaris/core/catalog/{PaginationToken.java => PageToken.java} (82%) diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index ecc9d45e1..142465e92 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -48,7 +48,7 @@ import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; -import org.apache.polaris.core.catalog.PaginationToken; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -524,7 +524,7 @@ public List lookupEntityActiveBatch( catalogId, parentId, entityType, - PaginationToken.readEverything(), + PageToken.readEverything(), entityFilter, entity -> new PolarisEntityActiveRecord( @@ -542,7 +542,7 @@ public List lookupEntityActiveBatch( long catalogId, long parentId, @NotNull PolarisEntityType entityType, - @NotNull PaginationToken paginationToken, + @NotNull PageToken pageToken, @NotNull Predicate entityFilter, @NotNull Function transformer) { // full range scan under the parent for that type diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PaginationToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java similarity index 82% rename from polaris-core/src/main/java/org/apache/polaris/core/catalog/PaginationToken.java rename to polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java index cfdfe75a9..a7b5f637c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PaginationToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java @@ -21,7 +21,7 @@ import java.util.Base64; -public class PaginationToken { +public class PageToken { public final long offset; public final long pageSize; @@ -30,7 +30,7 @@ public class PaginationToken { public static final long TOKEN_START = 0; public static final long DEFAULT_PAGE_SIZE = 1000; - public PaginationToken(long offset, long pageSize) { + public PageToken(long offset, long pageSize) { this.offset = offset; this.pageSize = pageSize; } @@ -38,23 +38,23 @@ public PaginationToken(long offset, long pageSize) { /** * Construct a PaginationToken from a plain limit */ - public static PaginationToken fromLimit(int limit) { - return new PaginationToken(TOKEN_START, limit); + public static PageToken fromLimit(int limit) { + return new PageToken(TOKEN_START, limit); } /** * Construct a PaginationToken from a plain limit */ - public static PaginationToken readEverything() { - return new PaginationToken(TOKEN_START, Integer.MAX_VALUE); + public static PageToken readEverything() { + return new PageToken(TOKEN_START, Integer.MAX_VALUE); } /** * Decode a token string into a PaginationToken object */ - public static PaginationToken fromString(String tokenString) { + public static PageToken fromString(String tokenString) { if (tokenString == null || tokenString.isEmpty()) { - return new PaginationToken(TOKEN_START, DEFAULT_PAGE_SIZE); + return new PageToken(TOKEN_START, DEFAULT_PAGE_SIZE); } if (!tokenString.startsWith(TOKEN_PREFIX)) { @@ -72,7 +72,7 @@ public static PaginationToken fromString(String tokenString) { long offset = Long.parseLong(parts[1]); long pageSize = Long.parseLong(parts[2]); - return new PaginationToken(offset, pageSize); + return new PageToken(offset, pageSize); } catch (Exception e) { throw new IllegalArgumentException("Failed to decode token: " + tokenString, e); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 560b1d3a2..b16eb2c9c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -34,7 +34,7 @@ import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.polaris.core.catalog.PaginationToken; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -1483,7 +1483,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str catalogId, catalogId, PolarisEntityType.CATALOG_ROLE, - PaginationToken.fromLimit(2), + PageToken.fromLimit(2), entity -> true, Function.identity()); @@ -2015,7 +2015,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @NotNull PolarisCallContext callCtx, @NotNull PolarisMetaStoreSession ms, String executorId, - PaginationToken paginationToken) { + PageToken pageToken) { // find all available tasks List availableTasks = @@ -2024,7 +2024,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( PolarisEntityConstants.getRootEntityId(), PolarisEntityConstants.getRootEntityId(), PolarisEntityType.TASK, - paginationToken, + pageToken, entity -> { PolarisObjectMapperUtil.TaskExecutionState taskState = PolarisObjectMapperUtil.parseTaskState(entity); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index 23899a3b1..f29ff7424 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -23,7 +23,7 @@ import java.util.function.Predicate; import java.util.function.Supplier; -import org.apache.polaris.core.catalog.PaginationToken; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; @@ -341,7 +341,7 @@ List listActiveEntities( * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level * @param parentId id of the parent, can be the special 0 value representing the root entity * @param entityType type of entities to list - * @param paginationToken the pagination token to use + * @param pageToken the pagination token to use * @param entityFilter the filter to be applied to each entity. Only entities where the predicate * returns true are returned in the list * @param transformer the transformation function applied to the {@link PolarisBaseEntity} before @@ -354,7 +354,7 @@ List listActiveEntities( long catalogId, long parentId, @NotNull PolarisEntityType entityType, - @NotNull PaginationToken paginationToken, + @NotNull PageToken pageToken, @NotNull Predicate entityFilter, @NotNull Function transformer); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java index 939f12397..a84b0aadf 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java @@ -25,7 +25,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import org.apache.polaris.core.catalog.PaginationToken; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; @@ -342,7 +342,7 @@ public List lookupEntityActiveBatch( catalogId, parentId, entityType, - PaginationToken.readEverything(), + PageToken.readEverything(), entityFilter, entity -> new PolarisEntityActiveRecord( @@ -360,7 +360,7 @@ public List lookupEntityActiveBatch( long catalogId, long parentId, @NotNull PolarisEntityType entityType, - PaginationToken paginationToken, + PageToken pageToken, @NotNull Predicate entityFilter, @NotNull Function transformer) { // full range scan under the parent for that type @@ -369,8 +369,8 @@ public List lookupEntityActiveBatch( .readRange(this.store.buildPrefixKeyComposite(catalogId, parentId, entityType.getCode())) .stream() .filter(entityFilter) - .skip(paginationToken.offset) - .limit(paginationToken.pageSize) + .skip(pageToken.offset) + .limit(pageToken.pageSize) .map(transformer) .collect(Collectors.toList()); } From 609e7bf99d6212217f666fb2de1c0987ca38ea9c Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 5 Sep 2024 15:01:43 -0700 Subject: [PATCH 03/32] make tests stable --- .../java/org/apache/polaris/core/catalog/PageToken.java | 5 +++-- .../core/persistence/PolarisMetaStoreManager.java | 3 ++- .../core/persistence/PolarisMetaStoreManagerImpl.java | 4 ++-- .../polaris/service/catalog/BasePolarisCatalogTest.java | 5 +++-- .../service/task/TableCleanupTaskHandlerTest.java | 9 +++++---- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java index a7b5f637c..10203e4b0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java @@ -19,6 +19,7 @@ package org.apache.polaris.core.catalog; +import java.nio.charset.StandardCharsets; import java.util.Base64; public class PageToken { @@ -62,7 +63,7 @@ public static PageToken fromString(String tokenString) { } try { - String decoded = new String(Base64.getDecoder().decode(tokenString)); + String decoded = new String(Base64.getDecoder().decode(tokenString), StandardCharsets.UTF_8); String[] parts = decoded.split(":"); if (parts.length != 3 || !parts[0].equals(TOKEN_PREFIX)) { @@ -81,6 +82,6 @@ public static PageToken fromString(String tokenString) { @Override public String toString() { String tokenContent = TOKEN_PREFIX + ":" + offset + ":" + pageSize; - return Base64.getEncoder().encodeToString(tokenContent.getBytes()); + return Base64.getEncoder().encodeToString(tokenContent.getBytes(StandardCharsets.UTF_8)); } } \ No newline at end of file diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java index 9033795d8..845775558 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java @@ -27,6 +27,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntity; @@ -1199,7 +1200,7 @@ ChangeTrackingResult loadEntitiesChangeTracking( * @return list of tasks to be completed */ @NotNull - EntitiesResult loadTasks(@NotNull PolarisCallContext callCtx, String executorId, int limit); + EntitiesResult loadTasks(@NotNull PolarisCallContext callCtx, String executorId, PageToken pageToken); /** Result of a getSubscopedCredsForEntity() call */ class ScopedCredentialsResult extends BaseResult { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index b16eb2c9c..5a7209f5f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -2063,9 +2063,9 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @Override public @NotNull EntitiesResult loadTasks( - @NotNull PolarisCallContext callCtx, String executorId, int limit) { + @NotNull PolarisCallContext callCtx, String executorId, PageToken pageToken) { PolarisMetaStoreSession ms = callCtx.getMetaStore(); - return ms.runInTransaction(callCtx, () -> this.loadTasks(callCtx, ms, executorId, limit)); + return ms.runInTransaction(callCtx, () -> this.loadTasks(callCtx, ms, executorId, pageToken)); } /** {@inheritDoc} */ diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index c574e5f42..f951611d3 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -62,6 +62,7 @@ import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.CatalogEntity; @@ -1121,7 +1122,7 @@ public void testDropTableWithPurge() { .as("Table should not exist after drop") .rejects(TABLE); List tasks = - metaStoreManager.loadTasks(polarisContext, "testExecutor", 1).getEntities(); + metaStoreManager.loadTasks(polarisContext, "testExecutor", PageToken.fromLimit(1)).getEntities(); Assertions.assertThat(tasks).hasSize(1); TaskEntity taskEntity = TaskEntity.of(tasks.get(0)); EnumMap credentials = @@ -1235,7 +1236,7 @@ public void testFileIOWrapper() { handler.handleTask( TaskEntity.of( metaStoreManager - .loadTasks(polarisContext, "testExecutor", 1) + .loadTasks(polarisContext, "testExecutor", PageToken.fromLimit(1)) .getEntities() .getFirst())); Assertions.assertThat(measured.getNumDeletedFiles()).as("A table was deleted").isGreaterThan(0); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java b/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java index d106a26e0..75e48dec8 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java @@ -32,6 +32,7 @@ import org.apache.iceberg.io.FileIO; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.AsyncTaskType; @@ -91,7 +92,7 @@ public void testTableCleanup() throws IOException { assertThat( metaStoreManagerFactory .getOrCreateMetaStoreManager(realmContext) - .loadTasks(polarisCallContext, "test", 1) + .loadTasks(polarisCallContext, "test", PageToken.fromLimit(1)) .getEntities()) .hasSize(1) .satisfiesExactly( @@ -166,7 +167,7 @@ public void close() { assertThat( metaStoreManagerFactory .getOrCreateMetaStoreManager(realmContext) - .loadTasks(polarisCallContext, "test", 5) + .loadTasks(polarisCallContext, "test", PageToken.fromLimit(5)) .getEntities()) .hasSize(1); } @@ -236,7 +237,7 @@ public void close() { assertThat( metaStoreManagerFactory .getOrCreateMetaStoreManager(realmContext) - .loadTasks(polarisCallContext, "test", 5) + .loadTasks(polarisCallContext, "test", PageToken.fromLimit(5)) .getEntities()) .hasSize(2) .satisfiesExactly( @@ -326,7 +327,7 @@ public void testTableCleanupMultipleSnapshots() throws IOException { assertThat( metaStoreManagerFactory .getOrCreateMetaStoreManager(realmContext) - .loadTasks(polarisCallContext, "test", 5) + .loadTasks(polarisCallContext, "test", PageToken.fromLimit(5)) .getEntities()) // all three manifests should be present, even though one is excluded from the latest // snapshot From 964bd53ee5e69c6ba5afd2b5f730ce39adbefa40 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 5 Sep 2024 17:54:32 -0700 Subject: [PATCH 04/32] still chasing compile errors --- .../polaris/core/catalog/PageToken.java | 19 ++++++-- .../polaris/core/catalog/PolarisPage.java | 37 ++++++++++++++++ .../persistence/PolarisMetaStoreManager.java | 43 ++++++++++++++++--- .../PolarisMetaStoreManagerImpl.java | 27 ++++++------ .../persistence/PolarisMetaStoreSession.java | 7 +-- .../PolarisTreeMapMetaStoreSessionImpl.java | 10 +++-- .../service/catalog/BasePolarisCatalog.java | 30 +++++++------ 7 files changed, 132 insertions(+), 41 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/catalog/PolarisPage.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java index 10203e4b0..f31593952 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java @@ -21,15 +21,21 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; +import java.util.List; +/** + * Represents a page token + */ public class PageToken { public final long offset; public final long pageSize; - public static final String TOKEN_PREFIX = "polaris"; - public static final long TOKEN_START = 0; - public static final long DEFAULT_PAGE_SIZE = 1000; + private static final String TOKEN_PREFIX = "polaris"; + private static final long TOKEN_START = 0; + private static final long DEFAULT_PAGE_SIZE = 1000; + + public static PageToken DONE = null; public PageToken(long offset, long pageSize) { this.offset = offset; @@ -79,6 +85,13 @@ public static PageToken fromString(String tokenString) { } } + /** + * Builds a new page token to reflect new data that's been read + */ + public PageToken updated(List newData) { + return new PageToken(offset + newData.size(), pageSize); + } + @Override public String toString() { String tokenContent = TOKEN_PREFIX + ":" + offset + ":" + pageSize; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PolarisPage.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PolarisPage.java new file mode 100644 index 000000000..91d92edfb --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PolarisPage.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.catalog; + +import java.util.List; + +public class PolarisPage { + public final PageToken pageToken; + public final List data; + + public PolarisPage(PageToken pageToken, List data) { + this.pageToken = pageToken; + this.data = data; + } + + /** Used to wrap a List of data into a PolarisPage when there is no more data */ + public static PolarisPage fromData(List data) { + return new PolarisPage<>(PageToken.DONE, data); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java index 845775558..a032c1080 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java @@ -24,10 +24,12 @@ import java.util.EnumMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.catalog.PageToken; +import org.apache.polaris.core.catalog.PolarisPage; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntity; @@ -310,6 +312,11 @@ class ListEntitiesResult extends BaseResult { // null if not success. Else the list of entities being returned private final List entities; + private final Optional pageTokenOpt; + + public static ListEntitiesResult fromPolarisPage(PolarisPage polarisPage) { + return new ListEntitiesResult(polarisPage.data, Optional.of(polarisPage.pageToken)); + } /** * Constructor for an error @@ -319,9 +326,11 @@ class ListEntitiesResult extends BaseResult { */ public ListEntitiesResult( @NotNull PolarisMetaStoreManager.ReturnStatus errorCode, - @Nullable String extraInformation) { + @Nullable String extraInformation, + @NotNull Optional pageTokenOpt) { super(errorCode, extraInformation); this.entities = null; + this.pageTokenOpt = pageTokenOpt; } /** @@ -329,23 +338,32 @@ public ListEntitiesResult( * * @param entities list of entities being returned, implies success */ - public ListEntitiesResult(@NotNull List entities) { + public ListEntitiesResult( + @NotNull List entities, + @Nullable Optional pageTokenOpt) { super(ReturnStatus.SUCCESS); this.entities = entities; + this.pageTokenOpt = pageTokenOpt; } @JsonCreator private ListEntitiesResult( @JsonProperty("returnStatus") @NotNull ReturnStatus returnStatus, @JsonProperty("extraInformation") String extraInformation, - @JsonProperty("entities") List entities) { + @JsonProperty("entities") List entities, + @JsonProperty("pageToken") Optional pageTokenOpt) { super(returnStatus, extraInformation); this.entities = entities; + this.pageTokenOpt = pageTokenOpt; } public List getEntities() { return entities; } + + public Optional getPageToken() { + return pageTokenOpt; + } } /** @@ -664,6 +682,11 @@ class EntitiesResult extends BaseResult { // null if not success. Else the list of entities being returned private final List entities; + private final Optional pageTokenOpt; + + public static EntitiesResult fromPolarisPage(PolarisPage polarisPage) { + return new EntitiesResult(polarisPage.data, Optional.of(polarisPage.pageToken)); + } /** * Constructor for an error @@ -676,6 +699,7 @@ public EntitiesResult( @Nullable String extraInformation) { super(errorStatus, extraInformation); this.entities = null; + this.pageTokenOpt = Optional.empty(); } /** @@ -683,23 +707,32 @@ public EntitiesResult( * * @param entities list of entities being returned, implies success */ - public EntitiesResult(@NotNull List entities) { + public EntitiesResult( + @NotNull List entities, + @NotNull Optional pageTokenOpt) { super(ReturnStatus.SUCCESS); this.entities = entities; + this.pageTokenOpt = pageTokenOpt; } @JsonCreator private EntitiesResult( @JsonProperty("returnStatus") @NotNull ReturnStatus returnStatus, @JsonProperty("extraInformation") String extraInformation, - @JsonProperty("entities") List entities) { + @JsonProperty("entities") List entities, + @JsonProperty("pageToken") Optional pageTokenOpt) { super(returnStatus, extraInformation); this.entities = entities; + this.pageTokenOpt = pageTokenOpt; } public List getEntities() { return entities; } + + public Optional getPageToken() { + return pageTokenOpt; + } } /** diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 5a7209f5f..01fd697d9 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -30,12 +30,14 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.catalog.PolarisPage; import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; @@ -793,24 +795,25 @@ private void bootstrapPolarisService( // return if we failed to resolve if (resolver.isFailure()) { - return new ListEntitiesResult(ReturnStatus.CATALOG_PATH_CANNOT_BE_RESOLVED, null); + return new ListEntitiesResult(ReturnStatus.CATALOG_PATH_CANNOT_BE_RESOLVED, null, Optional.empty()); } // return list of active entities - List toreturnList = + PolarisPage resultPage = ms.listActiveEntities( callCtx, resolver.getCatalogIdOrNull(), resolver.getParentId(), entityType); // prune the returned list with only entities matching the entity subtype if (entitySubType != PolarisEntitySubType.ANY_SUBTYPE) { - toreturnList = - toreturnList.stream() - .filter(rec -> rec.getSubTypeCode() == entitySubType.getCode()) - .collect(Collectors.toList()); + resultPage = new PolarisPage( + resultPage.pageToken, + resultPage.data.stream() + .filter(rec -> rec.getSubTypeCode() == entitySubType.getCode()) + .collect(Collectors.toList())); } // done - return new ListEntitiesResult(toreturnList); + return ListEntitiesResult.fromPolarisPage(resultPage); } /** {@inheritDoc} */ @@ -1199,7 +1202,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str } createdEntities.add(entityCreateResult.getEntity()); } - return new EntitiesResult(createdEntities); + return new EntitiesResult(new PolarisPage<>(createdEntities)); }); } @@ -1485,7 +1488,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str PolarisEntityType.CATALOG_ROLE, PageToken.fromLimit(2), entity -> true, - Function.identity()); + Function.identity()).data; // if we have 2, we cannot drop the catalog. If only one left, better be the admin role if (catalogRoles.size() > 1) { @@ -2018,7 +2021,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( PageToken pageToken) { // find all available tasks - List availableTasks = + PolarisPage availableTasks = ms.listActiveEntities( callCtx, PolarisEntityConstants.getRootEntityId(), @@ -2041,7 +2044,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( }, Function.identity()); - availableTasks.forEach( + availableTasks.data.forEach( task -> { Map properties = PolarisObjectMapperUtil.deserializeProperties(callCtx, task.getProperties()); @@ -2058,7 +2061,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( task.setProperties(PolarisObjectMapperUtil.serializeProperties(callCtx, properties)); writeEntity(callCtx, ms, task, false); }); - return new EntitiesResult(availableTasks); + return EntitiesResult.fromPolarisPage(availableTasks); } @Override diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index f29ff7424..d888dfad5 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -25,6 +25,7 @@ import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.catalog.PolarisPage; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; @@ -308,7 +309,7 @@ List lookupEntityActiveBatch( * @return the list of entities_active records for the specified list operation */ @NotNull - List listActiveEntities( + PolarisPage listActiveEntities( @NotNull PolarisCallContext callCtx, long catalogId, long parentId, @@ -326,7 +327,7 @@ List listActiveEntities( * @return the list of entities for which the predicate returns true */ @NotNull - List listActiveEntities( + PolarisPage listActiveEntities( @NotNull PolarisCallContext callCtx, long catalogId, long parentId, @@ -349,7 +350,7 @@ List listActiveEntities( * @return the list of entities for which the predicate returns true */ @NotNull - List listActiveEntities( + PolarisPage listActiveEntities( @NotNull PolarisCallContext callCtx, long catalogId, long parentId, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java index a84b0aadf..707612ef9 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java @@ -27,6 +27,7 @@ import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.catalog.PolarisPage; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; @@ -321,7 +322,7 @@ public List lookupEntityActiveBatch( /** {@inheritDoc} */ @Override - public @NotNull List listActiveEntities( + public @NotNull PolarisPage listActiveEntities( @NotNull PolarisCallContext callCtx, long catalogId, long parentId, @@ -330,7 +331,7 @@ public List lookupEntityActiveBatch( } @Override - public @NotNull List listActiveEntities( + public @NotNull PolarisPage listActiveEntities( @NotNull PolarisCallContext callCtx, long catalogId, long parentId, @@ -355,7 +356,7 @@ public List lookupEntityActiveBatch( } @Override - public @NotNull List listActiveEntities( + public @NotNull PolarisPage listActiveEntities( @NotNull PolarisCallContext callCtx, long catalogId, long parentId, @@ -364,7 +365,7 @@ public List lookupEntityActiveBatch( @NotNull Predicate entityFilter, @NotNull Function transformer) { // full range scan under the parent for that type - return this.store + List entities = this.store .getSliceEntitiesActive() .readRange(this.store.buildPrefixKeyComposite(catalogId, parentId, entityType.getCode())) .stream() @@ -373,6 +374,7 @@ public List lookupEntityActiveBatch( .limit(pageToken.pageSize) .map(transformer) .collect(Collectors.toList()); + return new PolarisPage(new PageToken(pageToken.offset + entities.size(), pageToken.pageSize), entities); } /** {@inheritDoc} */ diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 2538caf50..7b7961746 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -29,6 +29,7 @@ import java.io.Closeable; import java.io.IOException; import java.net.URI; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -79,6 +80,7 @@ import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; +import org.apache.polaris.core.catalog.PolarisPage; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.NamespaceEntity; @@ -448,7 +450,7 @@ public boolean dropTable(TableIdentifier tableIdentifier, boolean purge) { } @Override - public List listTables(Namespace namespace) { + public PolarisPage listTables(Namespace namespace) { if (!namespaceExists(namespace) && !namespace.isEmpty()) { throw new NoSuchNamespaceException( "Cannot list tables for namespace. Namespace does not exist: %s", namespace); @@ -772,7 +774,7 @@ public List listViews(Namespace namespace) { "Cannot list views for namespace. Namespace does not exist: %s", namespace); } - return listTableLike(PolarisEntitySubType.VIEW, namespace); + return listTableLike(PolarisEntitySubType.VIEW, namespace).data; } @Override @@ -1935,7 +1937,7 @@ private void createNonExistingNamespaces(Namespace namespace) { } } - private List listTableLike(PolarisEntitySubType subType, Namespace namespace) { + private PolarisPage listTableLike(PolarisEntitySubType subType, Namespace namespace) { PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(namespace); if (resolvedEntities == null) { // Illegal state because the namespace should've already been in the static resolution set. @@ -1944,17 +1946,17 @@ private List listTableLike(PolarisEntitySubType subType, Namesp } List catalogPath = resolvedEntities.getRawFullPath(); - List entities = - PolarisEntity.toNameAndIdList( - entityManager - .getMetaStoreManager() - .listEntities( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(catalogPath), - PolarisEntityType.TABLE_LIKE, - subType) - .getEntities()); - return PolarisCatalogHelpers.nameAndIdToTableIdentifiers(catalogPath, entities); + PolarisMetaStoreManager.ListEntitiesResult listResult = + entityManager + .getMetaStoreManager() + .listEntities( + getCurrentPolarisContext(), + PolarisEntity.toCoreList(catalogPath), + PolarisEntityType.TABLE_LIKE, + subType); + List entities = PolarisEntity.toNameAndIdList(listResult.getEntities()); + List identifiers = PolarisCatalogHelpers.nameAndIdToTableIdentifiers(catalogPath, entities); + return PolarisPage.fromData(identifiers); } /** From 0ce73af1986fd9be2407757ce3ce66529bf4853b Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 6 Sep 2024 01:12:29 -0700 Subject: [PATCH 05/32] tests stabler --- .../PolarisEclipseLinkMetaStoreSessionImpl.java | 15 ++++++++------- .../persistence/PolarisMetaStoreManagerImpl.java | 4 ++-- .../BasePolarisMetaStoreManagerTest.java | 15 ++++++++------- .../service/catalog/BasePolarisCatalog.java | 4 ++-- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index 142465e92..e6e2a84b1 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -50,6 +50,7 @@ import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.catalog.PolarisPage; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; @@ -503,7 +504,7 @@ public List lookupEntityActiveBatch( /** {@inheritDoc} */ @Override - public @NotNull List listActiveEntities( + public @NotNull PolarisPage listActiveEntities( @NotNull PolarisCallContext callCtx, long catalogId, long parentId, @@ -512,7 +513,7 @@ public List lookupEntityActiveBatch( } @Override - public @NotNull List listActiveEntities( + public @NotNull PolarisPage listActiveEntities( @NotNull PolarisCallContext callCtx, long catalogId, long parentId, @@ -537,7 +538,7 @@ public List lookupEntityActiveBatch( } @Override - public @NotNull List listActiveEntities( + public @NotNull PolarisPage listActiveEntities( @NotNull PolarisCallContext callCtx, long catalogId, long parentId, @@ -545,16 +546,16 @@ public List lookupEntityActiveBatch( @NotNull PageToken pageToken, @NotNull Predicate entityFilter, @NotNull Function transformer) { - // full range scan under the parent for that type - return this.store + List data = this.store .lookupFullEntitiesActive(localSession.get(), catalogId, parentId, entityType) .stream() .map(ModelEntity::toEntity) .filter(entityFilter) -// .skip(paginationToken.offset()) -// .limit(paginationToken.pageSize()) + .skip(pageToken.offset) + .limit(pageToken.pageSize) .map(transformer) .collect(Collectors.toList()); + return new PolarisPage(pageToken.updated(data), data); } /** {@inheritDoc} */ diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 01fd697d9..9a54f8c33 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -1202,7 +1202,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str } createdEntities.add(entityCreateResult.getEntity()); } - return new EntitiesResult(new PolarisPage<>(createdEntities)); + return new EntitiesResult(createdEntities, Optional.empty()); }); } @@ -1291,7 +1291,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str } // good, all success - return new EntitiesResult(updatedEntities); + return new EntitiesResult(updatedEntities, Optional.empty()); } /** {@inheritDoc} */ diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java index bf97e4128..80b73aa73 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java @@ -35,6 +35,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -290,7 +291,7 @@ void testLoadTasks() { PolarisMetaStoreManager metaStoreManager = polarisTestMetaStoreManager.polarisMetaStoreManager; PolarisCallContext callCtx = polarisTestMetaStoreManager.polarisCallContext; List taskList = - metaStoreManager.loadTasks(callCtx, executorId, 5).getEntities(); + metaStoreManager.loadTasks(callCtx, executorId, PageToken.fromLimit(5)).getEntities(); Assertions.assertThat(taskList) .isNotNull() .isNotEmpty() @@ -310,7 +311,7 @@ void testLoadTasks() { // grab a second round of tasks. Assert that none of the original 5 are in the list List newTaskList = - metaStoreManager.loadTasks(callCtx, executorId, 5).getEntities(); + metaStoreManager.loadTasks(callCtx, executorId, PageToken.fromLimit(5)).getEntities(); Assertions.assertThat(newTaskList) .isNotNull() .isNotEmpty() @@ -324,7 +325,7 @@ void testLoadTasks() { // only 10 tasks are unassigned. Requesting 20, we should only receive those 10 List lastTen = - metaStoreManager.loadTasks(callCtx, executorId, 20).getEntities(); + metaStoreManager.loadTasks(callCtx, executorId, PageToken.fromLimit(20)).getEntities(); Assertions.assertThat(lastTen) .isNotNull() @@ -338,7 +339,7 @@ void testLoadTasks() { .collect(Collectors.toSet()); List emtpyList = - metaStoreManager.loadTasks(callCtx, executorId, 20).getEntities(); + metaStoreManager.loadTasks(callCtx, executorId, PageToken.fromLimit(20)).getEntities(); Assertions.assertThat(emtpyList).isNotNull().isEmpty(); @@ -346,7 +347,7 @@ void testLoadTasks() { // all the tasks are unassigned. Fetch them all List allTasks = - metaStoreManager.loadTasks(callCtx, executorId, 20).getEntities(); + metaStoreManager.loadTasks(callCtx, executorId, PageToken.fromLimit(20)).getEntities(); Assertions.assertThat(allTasks) .isNotNull() @@ -361,7 +362,7 @@ void testLoadTasks() { timeSource.add(Duration.ofMinutes(10)); List finalList = - metaStoreManager.loadTasks(callCtx, executorId, 20).getEntities(); + metaStoreManager.loadTasks(callCtx, executorId, PageToken.fromLimit(20)).getEntities(); Assertions.assertThat(finalList).isNotNull().isEmpty(); } @@ -390,7 +391,7 @@ void testLoadTasksInParallel() throws Exception { do { retry = false; try { - taskList = metaStoreManager.loadTasks(callCtx, executorId, 5).getEntities(); + taskList = metaStoreManager.loadTasks(callCtx, executorId, PageToken.fromLimit(5)).getEntities(); taskList.stream().map(PolarisBaseEntity::getName).forEach(taskNames::add); } catch (RetryOnConcurrencyException e) { retry = true; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 7b7961746..d457d7848 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -450,13 +450,13 @@ public boolean dropTable(TableIdentifier tableIdentifier, boolean purge) { } @Override - public PolarisPage listTables(Namespace namespace) { + public List listTables(Namespace namespace) { if (!namespaceExists(namespace) && !namespace.isEmpty()) { throw new NoSuchNamespaceException( "Cannot list tables for namespace. Namespace does not exist: %s", namespace); } - return listTableLike(PolarisEntitySubType.TABLE, namespace); + return listTableLike(PolarisEntitySubType.TABLE, namespace).data; } @Override From cb4a8e2055cdd752c70773230d2a769b636911b8 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 6 Sep 2024 09:17:41 -0700 Subject: [PATCH 06/32] wip --- .../apache/polaris/core/catalog/PageToken.java | 18 +++++++++++------- .../service/catalog/BasePolarisCatalog.java | 9 ++++++++- .../service/catalog/IcebergCatalogAdapter.java | 7 ++++++- .../catalog/PolarisCatalogHandlerWrapper.java | 15 +++++++++++++-- 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java index f31593952..828188ec2 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java @@ -28,16 +28,16 @@ */ public class PageToken { - public final long offset; - public final long pageSize; + public final int offset; + public final int pageSize; private static final String TOKEN_PREFIX = "polaris"; - private static final long TOKEN_START = 0; - private static final long DEFAULT_PAGE_SIZE = 1000; + private static final int TOKEN_START = 0; + private static final int DEFAULT_PAGE_SIZE = 1000; public static PageToken DONE = null; - public PageToken(long offset, long pageSize) { + public PageToken(int offset, int pageSize) { this.offset = offset; this.pageSize = pageSize; } @@ -76,8 +76,8 @@ public static PageToken fromString(String tokenString) { throw new IllegalArgumentException("Invalid token format"); } - long offset = Long.parseLong(parts[1]); - long pageSize = Long.parseLong(parts[2]); + int offset = Integer.parseInt(parts[1]); + int pageSize = Integer.parseInt(parts[2]); return new PageToken(offset, pageSize); } catch (Exception e) { @@ -92,6 +92,10 @@ public PageToken updated(List newData) { return new PageToken(offset + newData.size(), pageSize); } + public PageToken withPageSize(int pageSize) { + return new PageToken(offset, pageSize); + } + @Override public String toString() { String tokenContent = TOKEN_PREFIX + ":" + offset + ":" + pageSize; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index d457d7848..ef796345b 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -79,6 +79,7 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; import org.apache.polaris.core.catalog.PolarisPage; import org.apache.polaris.core.context.CallContext; @@ -451,12 +452,18 @@ public boolean dropTable(TableIdentifier tableIdentifier, boolean purge) { @Override public List listTables(Namespace namespace) { + return listTables(namespace, PageToken.readEverything()).data; + } + + public PolarisPage listTables( + Namespace namespace, + PageToken pageToken) { if (!namespaceExists(namespace) && !namespace.isEmpty()) { throw new NoSuchNamespaceException( "Cannot list tables for namespace. Namespace does not exist: %s", namespace); } - return listTableLike(PolarisEntitySubType.TABLE, namespace).data; + return listTableLike(PolarisEntitySubType.TABLE, namespace); } @Override diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 5d15fc538..fc2105b89 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -49,6 +49,7 @@ import org.apache.iceberg.rest.responses.ConfigResponse; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisEntity; @@ -212,7 +213,11 @@ public Response listTables( Integer pageSize, SecurityContext securityContext) { Namespace ns = decodeNamespace(namespace); - return Response.ok(newHandlerWrapper(securityContext, prefix).listTables(ns)).build(); + + PageToken token = PageToken.fromString(pageToken).withPageSize(pageSize); + newHandlerWrapper(securityContext, prefix).listTables(ns, token); + + return Response.ok().build(); } @Override diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java index dfa552f52..047580936 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java @@ -71,7 +71,9 @@ import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizableOperation; import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; +import org.apache.polaris.core.catalog.PolarisPage; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; @@ -522,11 +524,20 @@ public UpdateNamespacePropertiesResponse updateNamespaceProperties( () -> CatalogHandlers.updateNamespaceProperties(namespaceCatalog, namespace, request)); } - public ListTablesResponse listTables(Namespace namespace) { + public ListTablesResponse listTables(Namespace namespace, PageToken pageToken) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_TABLES; authorizeBasicNamespaceOperationOrThrow(op, namespace); - return doCatalogOperation(() -> CatalogHandlers.listTables(baseCatalog, namespace)); + if (baseCatalog instanceof BasePolarisCatalog bpc) { + PolarisPage result = doCatalogOperation(() -> bpc.listTables(namespace, pageToken)); + return ListTablesResponse + .builder() + .addAll(result.data) + .build(); + // TODO where to add the next token header here? + } else { + return doCatalogOperation(() -> CatalogHandlers.listTables(baseCatalog, namespace)); + } } public LoadTableResponse createTableDirect(Namespace namespace, CreateTableRequest request) { From fb21e9be04e69634ce5026cf035dca25498147c8 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 6 Sep 2024 09:55:28 -0700 Subject: [PATCH 07/32] finishing integration --- .../polaris/core/catalog/PageToken.java | 27 ++++++-- .../catalog/IcebergCatalogAdapter.java | 8 ++- .../catalog/PolarisCatalogHandlerWrapper.java | 19 +++--- .../ListTablesResponseWithPageToken.java | 63 +++++++++++++++++++ 4 files changed, 101 insertions(+), 16 deletions(-) create mode 100644 polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java index 828188ec2..5a85f6bfd 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java @@ -64,10 +64,6 @@ public static PageToken fromString(String tokenString) { return new PageToken(TOKEN_START, DEFAULT_PAGE_SIZE); } - if (!tokenString.startsWith(TOKEN_PREFIX)) { - throw new IllegalArgumentException("Invalid token format"); - } - try { String decoded = new String(Base64.getDecoder().decode(tokenString), StandardCharsets.UTF_8); String[] parts = decoded.split(":"); @@ -92,8 +88,12 @@ public PageToken updated(List newData) { return new PageToken(offset + newData.size(), pageSize); } - public PageToken withPageSize(int pageSize) { - return new PageToken(offset, pageSize); + public PageToken withPageSize(Integer pageSize) { + if (pageSize == null) { + return this; + } else { + return new PageToken(offset, pageSize); + } } @Override @@ -101,4 +101,19 @@ public String toString() { String tokenContent = TOKEN_PREFIX + ":" + offset + ":" + pageSize; return Base64.getEncoder().encodeToString(tokenContent.getBytes(StandardCharsets.UTF_8)); } + + @Override + public boolean equals(Object o) { + if (o instanceof PageToken) { + PageToken other = (PageToken)o; + return offset == other.offset && pageSize == other.pageSize; + } else { + return false; + } + } + + @Override + public int hashCode() { + return offset + pageSize; + } } \ No newline at end of file diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index fc2105b89..0005edd9f 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -47,9 +47,11 @@ import org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest; import org.apache.iceberg.rest.requests.UpdateTableRequest; import org.apache.iceberg.rest.responses.ConfigResponse; +import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.catalog.PageToken; +import org.apache.polaris.core.catalog.PolarisPage; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisEntity; @@ -63,6 +65,7 @@ import org.apache.polaris.service.context.CallContextCatalogFactory; import org.apache.polaris.service.types.CommitTableRequest; import org.apache.polaris.service.types.CommitViewRequest; +import org.apache.polaris.service.types.ListTablesResponseWithPageToken; import org.apache.polaris.service.types.NotificationRequest; /** @@ -215,9 +218,10 @@ public Response listTables( Namespace ns = decodeNamespace(namespace); PageToken token = PageToken.fromString(pageToken).withPageSize(pageSize); - newHandlerWrapper(securityContext, prefix).listTables(ns, token); + PolarisPage result = newHandlerWrapper(securityContext, prefix).listTables(ns, token); + ListTablesResponseWithPageToken response = ListTablesResponseWithPageToken.fromPolarisPage(result); - return Response.ok().build(); + return Response.ok(response).build(); } @Override diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java index 047580936..b2ddd752d 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java @@ -524,22 +524,25 @@ public UpdateNamespacePropertiesResponse updateNamespaceProperties( () -> CatalogHandlers.updateNamespaceProperties(namespaceCatalog, namespace, request)); } - public ListTablesResponse listTables(Namespace namespace, PageToken pageToken) { + public PolarisPage listTables(Namespace namespace, PageToken pageToken) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_TABLES; authorizeBasicNamespaceOperationOrThrow(op, namespace); if (baseCatalog instanceof BasePolarisCatalog bpc) { - PolarisPage result = doCatalogOperation(() -> bpc.listTables(namespace, pageToken)); - return ListTablesResponse - .builder() - .addAll(result.data) - .build(); - // TODO where to add the next token header here? + return doCatalogOperation(() -> bpc.listTables(namespace, pageToken)); } else { - return doCatalogOperation(() -> CatalogHandlers.listTables(baseCatalog, namespace)); + return PolarisPage.fromData( + doCatalogOperation(() -> CatalogHandlers.listTables(baseCatalog, namespace)).identifiers()); } } + public ListTablesResponse listTables(Namespace namespace) { + PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_TABLES; + authorizeBasicNamespaceOperationOrThrow(op, namespace); + + return doCatalogOperation(() -> CatalogHandlers.listTables(baseCatalog, namespace)); + } + public LoadTableResponse createTableDirect(Namespace namespace, CreateTableRequest request) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_TABLE_DIRECT; authorizeCreateTableLikeUnderNamespaceOperationOrThrow( diff --git a/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java b/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java new file mode 100644 index 000000000..0f1077279 --- /dev/null +++ b/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.types; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.responses.ListTablesResponse; +import org.apache.polaris.core.catalog.PageToken; +import org.apache.polaris.core.catalog.PolarisPage; + +import java.util.List; + +public class ListTablesResponseWithPageToken extends ListTablesResponse { + @JsonProperty("next-page-token") + private final PageToken pageToken; + private final List identifiers; + + + public ListTablesResponseWithPageToken(PageToken pageToken, List identifiers) { + this.pageToken = pageToken; + this.identifiers = identifiers; + this.validate(); + } + + public static ListTablesResponseWithPageToken fromPolarisPage(PolarisPage polarisPage) { + return new ListTablesResponseWithPageToken(polarisPage.pageToken, polarisPage.data); + } + + public PageToken getPageToken() { + return pageToken; + } + + @Override + public List identifiers() { + return this.identifiers != null ? this.identifiers : List.of(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("identifiers", this.identifiers) + .add("pageToken", this.pageToken) + .toString(); + } +} From a725497d2889808a82410648d74525044c06c670 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 6 Sep 2024 11:03:56 -0700 Subject: [PATCH 08/32] many fixes --- ...olarisEclipseLinkMetaStoreSessionImpl.java | 22 +-- .../polaris/core/catalog/PageToken.java | 149 ++++++++---------- .../polaris/core/catalog/PolarisPage.java | 21 ++- .../persistence/PolarisMetaStoreManager.java | 12 +- .../PolarisMetaStoreManagerImpl.java | 43 ++--- .../persistence/PolarisMetaStoreSession.java | 7 +- .../PolarisTreeMapMetaStoreSessionImpl.java | 33 ++-- .../BasePolarisMetaStoreManagerTest.java | 5 +- .../service/catalog/BasePolarisCatalog.java | 66 +++++--- .../catalog/IcebergCatalogAdapter.java | 14 +- .../catalog/PolarisCatalogHandlerWrapper.java | 28 +++- .../ListNamespacesResponseWithPageToken.java | 62 ++++++++ .../ListTablesResponseWithPageToken.java | 67 ++++---- .../catalog/BasePolarisCatalogTest.java | 4 +- 14 files changed, 320 insertions(+), 213 deletions(-) create mode 100644 polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index e6e2a84b1..d910d0836 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -47,9 +47,8 @@ import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; - -import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.catalog.PolarisPage; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -546,15 +545,16 @@ public List lookupEntityActiveBatch( @NotNull PageToken pageToken, @NotNull Predicate entityFilter, @NotNull Function transformer) { - List data = this.store - .lookupFullEntitiesActive(localSession.get(), catalogId, parentId, entityType) - .stream() - .map(ModelEntity::toEntity) - .filter(entityFilter) - .skip(pageToken.offset) - .limit(pageToken.pageSize) - .map(transformer) - .collect(Collectors.toList()); + List data = + this.store + .lookupFullEntitiesActive(localSession.get(), catalogId, parentId, entityType) + .stream() + .map(ModelEntity::toEntity) + .filter(entityFilter) + .skip(pageToken.offset) + .limit(pageToken.pageSize) + .map(transformer) + .collect(Collectors.toList()); return new PolarisPage(pageToken.updated(data), data); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java index 5a85f6bfd..770f3c13b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java @@ -16,104 +16,93 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.core.catalog; import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.List; -/** - * Represents a page token - */ +/** Represents a page token */ public class PageToken { - public final int offset; - public final int pageSize; + public final int offset; + public final int pageSize; - private static final String TOKEN_PREFIX = "polaris"; - private static final int TOKEN_START = 0; - private static final int DEFAULT_PAGE_SIZE = 1000; + private static final String TOKEN_PREFIX = "polaris"; + private static final int TOKEN_START = 0; + private static final int DEFAULT_PAGE_SIZE = 1000; - public static PageToken DONE = null; + public static PageToken DONE = null; - public PageToken(int offset, int pageSize) { - this.offset = offset; - this.pageSize = pageSize; - } + public PageToken(int offset, int pageSize) { + this.offset = offset; + this.pageSize = pageSize; + } - /** - * Construct a PaginationToken from a plain limit - */ - public static PageToken fromLimit(int limit) { - return new PageToken(TOKEN_START, limit); - } + /** Construct a PaginationToken from a plain limit */ + public static PageToken fromLimit(int limit) { + return new PageToken(TOKEN_START, limit); + } - /** - * Construct a PaginationToken from a plain limit - */ - public static PageToken readEverything() { - return new PageToken(TOKEN_START, Integer.MAX_VALUE); - } + /** Construct a PaginationToken from a plain limit */ + public static PageToken readEverything() { + return new PageToken(TOKEN_START, Integer.MAX_VALUE); + } - /** - * Decode a token string into a PaginationToken object - */ - public static PageToken fromString(String tokenString) { - if (tokenString == null || tokenString.isEmpty()) { - return new PageToken(TOKEN_START, DEFAULT_PAGE_SIZE); - } - - try { - String decoded = new String(Base64.getDecoder().decode(tokenString), StandardCharsets.UTF_8); - String[] parts = decoded.split(":"); - - if (parts.length != 3 || !parts[0].equals(TOKEN_PREFIX)) { - throw new IllegalArgumentException("Invalid token format"); - } - - int offset = Integer.parseInt(parts[1]); - int pageSize = Integer.parseInt(parts[2]); - - return new PageToken(offset, pageSize); - } catch (Exception e) { - throw new IllegalArgumentException("Failed to decode token: " + tokenString, e); - } + /** Decode a token string into a PaginationToken object */ + public static PageToken fromString(String tokenString) { + if (tokenString == null || tokenString.isEmpty()) { + return new PageToken(TOKEN_START, DEFAULT_PAGE_SIZE); } - /** - * Builds a new page token to reflect new data that's been read - */ - public PageToken updated(List newData) { - return new PageToken(offset + newData.size(), pageSize); - } + try { + String decoded = new String(Base64.getDecoder().decode(tokenString), StandardCharsets.UTF_8); + String[] parts = decoded.split(":"); - public PageToken withPageSize(Integer pageSize) { - if (pageSize == null) { - return this; - } else { - return new PageToken(offset, pageSize); - } - } + if (parts.length != 3 || !parts[0].equals(TOKEN_PREFIX)) { + throw new IllegalArgumentException("Invalid token format"); + } - @Override - public String toString() { - String tokenContent = TOKEN_PREFIX + ":" + offset + ":" + pageSize; - return Base64.getEncoder().encodeToString(tokenContent.getBytes(StandardCharsets.UTF_8)); - } + int offset = Integer.parseInt(parts[1]); + int pageSize = Integer.parseInt(parts[2]); - @Override - public boolean equals(Object o) { - if (o instanceof PageToken) { - PageToken other = (PageToken)o; - return offset == other.offset && pageSize == other.pageSize; - } else { - return false; - } + return new PageToken(offset, pageSize); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to decode token: " + tokenString, e); } - - @Override - public int hashCode() { - return offset + pageSize; + } + + /** Builds a new page token to reflect new data that's been read */ + public PageToken updated(List newData) { + return new PageToken(offset + newData.size(), pageSize); + } + + public PageToken withPageSize(Integer pageSize) { + if (pageSize == null) { + return this; + } else { + return new PageToken(offset, pageSize); } -} \ No newline at end of file + } + + @Override + public String toString() { + String tokenContent = TOKEN_PREFIX + ":" + offset + ":" + pageSize; + return Base64.getEncoder().encodeToString(tokenContent.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public boolean equals(Object o) { + if (o instanceof PageToken) { + PageToken other = (PageToken) o; + return offset == other.offset && pageSize == other.pageSize; + } else { + return false; + } + } + + @Override + public int hashCode() { + return offset + pageSize; + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PolarisPage.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PolarisPage.java index 91d92edfb..ba58cf5fb 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PolarisPage.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PolarisPage.java @@ -16,22 +16,21 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.core.catalog; import java.util.List; public class PolarisPage { - public final PageToken pageToken; - public final List data; + public final PageToken pageToken; + public final List data; - public PolarisPage(PageToken pageToken, List data) { - this.pageToken = pageToken; - this.data = data; - } + public PolarisPage(PageToken pageToken, List data) { + this.pageToken = pageToken; + this.data = data; + } - /** Used to wrap a List of data into a PolarisPage when there is no more data */ - public static PolarisPage fromData(List data) { - return new PolarisPage<>(PageToken.DONE, data); - } + /** Used to wrap a List of data into a PolarisPage when there is no more data */ + public static PolarisPage fromData(List data) { + return new PolarisPage<>(PageToken.DONE, data); + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java index a032c1080..f7c528ea8 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java @@ -314,7 +314,8 @@ class ListEntitiesResult extends BaseResult { private final List entities; private final Optional pageTokenOpt; - public static ListEntitiesResult fromPolarisPage(PolarisPage polarisPage) { + public static ListEntitiesResult fromPolarisPage( + PolarisPage polarisPage) { return new ListEntitiesResult(polarisPage.data, Optional.of(polarisPage.pageToken)); } @@ -383,7 +384,8 @@ ListEntitiesResult listEntities( @NotNull PolarisCallContext callCtx, @Nullable List catalogPath, @NotNull PolarisEntityType entityType, - @NotNull PolarisEntitySubType entitySubType); + @NotNull PolarisEntitySubType entitySubType, + @NotNull PageToken pageToken); /** the return for a generate new entity id */ class GenerateEntityIdResult extends BaseResult { @@ -708,8 +710,7 @@ public EntitiesResult( * @param entities list of entities being returned, implies success */ public EntitiesResult( - @NotNull List entities, - @NotNull Optional pageTokenOpt) { + @NotNull List entities, @NotNull Optional pageTokenOpt) { super(ReturnStatus.SUCCESS); this.entities = entities; this.pageTokenOpt = pageTokenOpt; @@ -1233,7 +1234,8 @@ ChangeTrackingResult loadEntitiesChangeTracking( * @return list of tasks to be completed */ @NotNull - EntitiesResult loadTasks(@NotNull PolarisCallContext callCtx, String executorId, PageToken pageToken); + EntitiesResult loadTasks( + @NotNull PolarisCallContext callCtx, String executorId, PageToken pageToken); /** Result of a getSubscopedCredsForEntity() call */ class ScopedCredentialsResult extends BaseResult { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 9a54f8c33..2f6d0206e 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -34,9 +34,8 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; - -import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.catalog.PolarisPage; import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -789,27 +788,30 @@ private void bootstrapPolarisService( @NotNull PolarisMetaStoreSession ms, @Nullable List catalogPath, @NotNull PolarisEntityType entityType, - @NotNull PolarisEntitySubType entitySubType) { + @NotNull PolarisEntitySubType entitySubType, + @NotNull PageToken pageToken) { // first resolve again the catalogPath to that entity PolarisEntityResolver resolver = new PolarisEntityResolver(callCtx, ms, catalogPath); // return if we failed to resolve if (resolver.isFailure()) { - return new ListEntitiesResult(ReturnStatus.CATALOG_PATH_CANNOT_BE_RESOLVED, null, Optional.empty()); + return new ListEntitiesResult( + ReturnStatus.CATALOG_PATH_CANNOT_BE_RESOLVED, null, Optional.empty()); } // return list of active entities PolarisPage resultPage = ms.listActiveEntities( - callCtx, resolver.getCatalogIdOrNull(), resolver.getParentId(), entityType); + callCtx, resolver.getCatalogIdOrNull(), resolver.getParentId(), entityType, pageToken); // prune the returned list with only entities matching the entity subtype if (entitySubType != PolarisEntitySubType.ANY_SUBTYPE) { - resultPage = new PolarisPage( - resultPage.pageToken, - resultPage.data.stream() - .filter(rec -> rec.getSubTypeCode() == entitySubType.getCode()) - .collect(Collectors.toList())); + resultPage = + new PolarisPage( + resultPage.pageToken, + resultPage.data.stream() + .filter(rec -> rec.getSubTypeCode() == entitySubType.getCode()) + .collect(Collectors.toList())); } // done @@ -822,13 +824,15 @@ private void bootstrapPolarisService( @NotNull PolarisCallContext callCtx, @Nullable List catalogPath, @NotNull PolarisEntityType entityType, - @NotNull PolarisEntitySubType entitySubType) { + @NotNull PolarisEntitySubType entitySubType, + @NotNull PageToken pageToken) { // get meta store we should be using PolarisMetaStoreSession ms = callCtx.getMetaStore(); // run operation in a read transaction return ms.runInReadTransaction( - callCtx, () -> listEntities(callCtx, ms, catalogPath, entityType, entitySubType)); + callCtx, + () -> listEntities(callCtx, ms, catalogPath, entityType, entitySubType, pageToken)); } /** {@inheritDoc} */ @@ -1482,13 +1486,14 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str // get the list of catalog roles, at most 2 List catalogRoles = ms.listActiveEntities( - callCtx, - catalogId, - catalogId, - PolarisEntityType.CATALOG_ROLE, - PageToken.fromLimit(2), - entity -> true, - Function.identity()).data; + callCtx, + catalogId, + catalogId, + PolarisEntityType.CATALOG_ROLE, + PageToken.fromLimit(2), + entity -> true, + Function.identity()) + .data; // if we have 2, we cannot drop the catalog. If only one left, better be the admin role if (catalogRoles.size() > 1) { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index d888dfad5..ee74f0730 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -22,9 +22,8 @@ import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; - -import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.catalog.PolarisPage; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; @@ -313,7 +312,8 @@ PolarisPage listActiveEntities( @NotNull PolarisCallContext callCtx, long catalogId, long parentId, - @NotNull PolarisEntityType entityType); + @NotNull PolarisEntityType entityType, + @NotNull PageToken pageToken); /** * List active entities where some predicate returns true @@ -332,6 +332,7 @@ PolarisPage listActiveEntities( long catalogId, long parentId, @NotNull PolarisEntityType entityType, + @NotNull PageToken pageToken, @NotNull Predicate entityFilter); /** diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java index 707612ef9..b4dfda055 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java @@ -24,9 +24,8 @@ import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; - -import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.catalog.PolarisPage; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; @@ -326,8 +325,10 @@ public List lookupEntityActiveBatch( @NotNull PolarisCallContext callCtx, long catalogId, long parentId, - @NotNull PolarisEntityType entityType) { - return listActiveEntities(callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue()); + @NotNull PolarisEntityType entityType, + @NotNull PageToken pageToken) { + return listActiveEntities( + callCtx, catalogId, parentId, entityType, pageToken, Predicates.alwaysTrue()); } @Override @@ -336,6 +337,7 @@ public List lookupEntityActiveBatch( long catalogId, long parentId, @NotNull PolarisEntityType entityType, + @NotNull PageToken pageToken, @NotNull Predicate entityFilter) { // full range scan under the parent for that type return listActiveEntities( @@ -365,16 +367,19 @@ public List lookupEntityActiveBatch( @NotNull Predicate entityFilter, @NotNull Function transformer) { // full range scan under the parent for that type - List entities = this.store - .getSliceEntitiesActive() - .readRange(this.store.buildPrefixKeyComposite(catalogId, parentId, entityType.getCode())) - .stream() - .filter(entityFilter) - .skip(pageToken.offset) - .limit(pageToken.pageSize) - .map(transformer) - .collect(Collectors.toList()); - return new PolarisPage(new PageToken(pageToken.offset + entities.size(), pageToken.pageSize), entities); + List entities = + this.store + .getSliceEntitiesActive() + .readRange( + this.store.buildPrefixKeyComposite(catalogId, parentId, entityType.getCode())) + .stream() + .filter(entityFilter) + .skip(pageToken.offset) + .limit(pageToken.pageSize) + .map(transformer) + .collect(Collectors.toList()); + return new PolarisPage( + new PageToken(pageToken.offset + entities.size(), pageToken.pageSize), entities); } /** {@inheritDoc} */ diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java index 80b73aa73..478790bc9 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java @@ -391,7 +391,10 @@ void testLoadTasksInParallel() throws Exception { do { retry = false; try { - taskList = metaStoreManager.loadTasks(callCtx, executorId, PageToken.fromLimit(5)).getEntities(); + taskList = + metaStoreManager + .loadTasks(callCtx, executorId, PageToken.fromLimit(5)) + .getEntities(); taskList.stream().map(PolarisBaseEntity::getName).forEach(taskNames::add); } catch (RetryOnConcurrencyException e) { retry = true; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index ef796345b..652669305 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -29,7 +29,6 @@ import java.io.Closeable; import java.io.IOException; import java.net.URI; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -455,15 +454,13 @@ public List listTables(Namespace namespace) { return listTables(namespace, PageToken.readEverything()).data; } - public PolarisPage listTables( - Namespace namespace, - PageToken pageToken) { + public PolarisPage listTables(Namespace namespace, PageToken pageToken) { if (!namespaceExists(namespace) && !namespace.isEmpty()) { throw new NoSuchNamespaceException( "Cannot list tables for namespace. Namespace does not exist: %s", namespace); } - return listTableLike(PolarisEntitySubType.TABLE, namespace); + return listTableLike(PolarisEntitySubType.TABLE, namespace, pageToken); } @Override @@ -752,23 +749,31 @@ public List listNamespaces() { @Override public List listNamespaces(Namespace namespace) throws NoSuchNamespaceException { + return listNamespaces(namespace, PageToken.readEverything()).data; + } + + public PolarisPage listNamespaces(Namespace namespace, PageToken pageToken) + throws NoSuchNamespaceException { PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(namespace); if (resolvedEntities == null) { throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); } List catalogPath = resolvedEntities.getRawFullPath(); - List entities = - PolarisEntity.toNameAndIdList( - entityManager - .getMetaStoreManager() - .listEntities( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(catalogPath), - PolarisEntityType.NAMESPACE, - PolarisEntitySubType.NULL_SUBTYPE) - .getEntities()); - return PolarisCatalogHelpers.nameAndIdToNamespaces(catalogPath, entities); + PolarisMetaStoreManager.ListEntitiesResult listResult = entityManager + .getMetaStoreManager() + .listEntities( + getCurrentPolarisContext(), + PolarisEntity.toCoreList(catalogPath), + PolarisEntityType.NAMESPACE, + PolarisEntitySubType.NULL_SUBTYPE, + pageToken); + List entities = PolarisEntity.toNameAndIdList(listResult.getEntities()); + List namespaces = PolarisCatalogHelpers.nameAndIdToNamespaces(catalogPath, entities); + + return listResult.getPageToken() + .map(token -> new PolarisPage<>(token, namespaces)) + .orElseGet(() -> PolarisPage.fromData(namespaces)); } @Override @@ -776,12 +781,16 @@ public void close() throws IOException {} @Override public List listViews(Namespace namespace) { + return listViews(namespace, PageToken.readEverything()); + } + + public List listViews(Namespace namespace, PageToken pageToken) { if (!namespaceExists(namespace) && !namespace.isEmpty()) { throw new NoSuchNamespaceException( "Cannot list views for namespace. Namespace does not exist: %s", namespace); } - return listTableLike(PolarisEntitySubType.VIEW, namespace).data; + return listTableLike(PolarisEntitySubType.VIEW, namespace, pageToken).data; } @Override @@ -1055,7 +1064,8 @@ private void validateNoLocationOverlap( callContext.getPolarisCallContext(), parentPath.stream().map(PolarisEntity::toCore).collect(Collectors.toList()), PolarisEntityType.NAMESPACE, - PolarisEntitySubType.ANY_SUBTYPE); + PolarisEntitySubType.ANY_SUBTYPE, + PageToken.readEverything()); if (!siblingNamespacesResult.isSuccess()) { throw new IllegalStateException( "Unable to resolve siblings entities to validate location - could not list namespaces"); @@ -1081,7 +1091,8 @@ private void validateNoLocationOverlap( .map(PolarisEntity::toCore) .collect(Collectors.toList()), PolarisEntityType.TABLE_LIKE, - PolarisEntitySubType.ANY_SUBTYPE); + PolarisEntitySubType.ANY_SUBTYPE, + PageToken.readEverything()); if (!siblingTablesResult.isSuccess()) { throw new IllegalStateException( "Unable to resolve siblings entities to validate location - could not list tables"); @@ -1944,7 +1955,8 @@ private void createNonExistingNamespaces(Namespace namespace) { } } - private PolarisPage listTableLike(PolarisEntitySubType subType, Namespace namespace) { + private PolarisPage listTableLike( + PolarisEntitySubType subType, Namespace namespace, PageToken pageToken) { PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(namespace); if (resolvedEntities == null) { // Illegal state because the namespace should've already been in the static resolution set. @@ -1960,10 +1972,16 @@ private PolarisPage listTableLike(PolarisEntitySubType subType, getCurrentPolarisContext(), PolarisEntity.toCoreList(catalogPath), PolarisEntityType.TABLE_LIKE, - subType); - List entities = PolarisEntity.toNameAndIdList(listResult.getEntities()); - List identifiers = PolarisCatalogHelpers.nameAndIdToTableIdentifiers(catalogPath, entities); - return PolarisPage.fromData(identifiers); + subType, + pageToken); + List entities = + PolarisEntity.toNameAndIdList(listResult.getEntities()); + List identifiers = + PolarisCatalogHelpers.nameAndIdToTableIdentifiers(catalogPath, entities); + + return listResult.getPageToken() + .map(token -> new PolarisPage<>(token, identifiers)) + .orElseGet(() -> PolarisPage.fromData(identifiers)); } /** diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 0005edd9f..7dec28997 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -47,7 +47,6 @@ import org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest; import org.apache.iceberg.rest.requests.UpdateTableRequest; import org.apache.iceberg.rest.responses.ConfigResponse; -import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.catalog.PageToken; @@ -129,9 +128,11 @@ public Response listNamespaces( SecurityContext securityContext) { Optional namespaceOptional = Optional.ofNullable(parent).map(IcebergCatalogAdapter::decodeNamespace); + + PageToken token = PageToken.fromString(pageToken).withPageSize(pageSize); return Response.ok( - newHandlerWrapper(securityContext, prefix) - .listNamespaces(namespaceOptional.orElse(Namespace.of()))) + newHandlerWrapper(securityContext, prefix) + .listNamespaces(namespaceOptional.orElse(Namespace.of()), token)) .build(); } @@ -218,10 +219,11 @@ public Response listTables( Namespace ns = decodeNamespace(namespace); PageToken token = PageToken.fromString(pageToken).withPageSize(pageSize); - PolarisPage result = newHandlerWrapper(securityContext, prefix).listTables(ns, token); - ListTablesResponseWithPageToken response = ListTablesResponseWithPageToken.fromPolarisPage(result); - return Response.ok(response).build(); + return Response.ok( + newHandlerWrapper(securityContext, prefix) + .listTables(ns, token)) + .build(); } @Override diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java index b2ddd752d..9e2724a4b 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java @@ -85,6 +85,8 @@ import org.apache.polaris.core.persistence.resolver.ResolverStatus; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.service.context.CallContextCatalogFactory; +import org.apache.polaris.service.types.ListNamespacesResponseWithPageToken; +import org.apache.polaris.service.types.ListTablesResponseWithPageToken; import org.apache.polaris.service.types.NotificationRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -411,6 +413,21 @@ private void authorizeRenameTableLikeOperationOrThrow( initializeCatalog(); } + public ListNamespacesResponseWithPageToken listNamespaces(Namespace parent, PageToken pageToken) { + PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_NAMESPACES; + authorizeBasicNamespaceOperationOrThrow(op, parent); + + if (baseCatalog instanceof BasePolarisCatalog bpc) { + ListNamespacesResponseWithPageToken + .fromPolarisPage(doCatalogOperation(() -> bpc.listNamespaces(parent, pageToken))); + } else { + return ListNamespacesResponseWithPageToken + .fromPolarisPage(PolarisPage.fromData( + doCatalogOperation(() -> CatalogHandlers.listNamespaces(namespaceCatalog, parent)) + .namespaces())); + } + } + public ListNamespacesResponse listNamespaces(Namespace parent) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_NAMESPACES; authorizeBasicNamespaceOperationOrThrow(op, parent); @@ -524,15 +541,18 @@ public UpdateNamespacePropertiesResponse updateNamespaceProperties( () -> CatalogHandlers.updateNamespaceProperties(namespaceCatalog, namespace, request)); } - public PolarisPage listTables(Namespace namespace, PageToken pageToken) { + public ListTablesResponseWithPageToken listTables(Namespace namespace, PageToken pageToken) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_TABLES; authorizeBasicNamespaceOperationOrThrow(op, namespace); if (baseCatalog instanceof BasePolarisCatalog bpc) { - return doCatalogOperation(() -> bpc.listTables(namespace, pageToken)); + return ListTablesResponseWithPageToken.fromPolarisPage( + doCatalogOperation(() -> bpc.listTables(namespace, pageToken))); } else { - return PolarisPage.fromData( - doCatalogOperation(() -> CatalogHandlers.listTables(baseCatalog, namespace)).identifiers()); + return ListTablesResponseWithPageToken.fromPolarisPage( + PolarisPage.fromData( + doCatalogOperation(() -> CatalogHandlers.listTables(baseCatalog, namespace)) + .identifiers())); } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java b/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java new file mode 100644 index 000000000..b5c0391bf --- /dev/null +++ b/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.types; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import java.util.List; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.rest.responses.ListNamespacesResponse; +import org.apache.polaris.core.catalog.PageToken; +import org.apache.polaris.core.catalog.PolarisPage; + +public class ListNamespacesResponseWithPageToken extends ListNamespacesResponse { + @JsonProperty("next-page-token") + private final PageToken pageToken; + + private final List namespaces; + + public ListNamespacesResponseWithPageToken(PageToken pageToken, List namespaces) { + this.pageToken = pageToken; + this.namespaces = namespaces; + this.validate(); + } + + public static ListNamespacesResponseWithPageToken fromPolarisPage( + PolarisPage polarisPage) { + return new ListNamespacesResponseWithPageToken(polarisPage.pageToken, polarisPage.data); + } + + public PageToken getPageToken() { + return pageToken; + } + + @Override + public List namespaces() { + return this.namespaces != null ? this.namespaces : List.of(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("namespaces", this.namespaces) + .add("pageToken", this.pageToken) + .toString(); + } +} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java b/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java index 0f1077279..d073ea202 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java @@ -16,48 +16,47 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.polaris.service.types; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.MoreObjects; +import java.util.List; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.catalog.PolarisPage; -import java.util.List; - public class ListTablesResponseWithPageToken extends ListTablesResponse { - @JsonProperty("next-page-token") - private final PageToken pageToken; - private final List identifiers; - - - public ListTablesResponseWithPageToken(PageToken pageToken, List identifiers) { - this.pageToken = pageToken; - this.identifiers = identifiers; - this.validate(); - } - - public static ListTablesResponseWithPageToken fromPolarisPage(PolarisPage polarisPage) { - return new ListTablesResponseWithPageToken(polarisPage.pageToken, polarisPage.data); - } - - public PageToken getPageToken() { - return pageToken; - } - - @Override - public List identifiers() { - return this.identifiers != null ? this.identifiers : List.of(); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("identifiers", this.identifiers) - .add("pageToken", this.pageToken) - .toString(); - } + @JsonProperty("next-page-token") + private final PageToken pageToken; + + private final List identifiers; + + public ListTablesResponseWithPageToken(PageToken pageToken, List identifiers) { + this.pageToken = pageToken; + this.identifiers = identifiers; + this.validate(); + } + + public static ListTablesResponseWithPageToken fromPolarisPage( + PolarisPage polarisPage) { + return new ListTablesResponseWithPageToken(polarisPage.pageToken, polarisPage.data); + } + + public PageToken getPageToken() { + return pageToken; + } + + @Override + public List identifiers() { + return this.identifiers != null ? this.identifiers : List.of(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("identifiers", this.identifiers) + .add("pageToken", this.pageToken) + .toString(); + } } diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index f951611d3..12371d31c 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -1122,7 +1122,9 @@ public void testDropTableWithPurge() { .as("Table should not exist after drop") .rejects(TABLE); List tasks = - metaStoreManager.loadTasks(polarisContext, "testExecutor", PageToken.fromLimit(1)).getEntities(); + metaStoreManager + .loadTasks(polarisContext, "testExecutor", PageToken.fromLimit(1)) + .getEntities(); Assertions.assertThat(tasks).hasSize(1); TaskEntity taskEntity = TaskEntity.of(tasks.get(0)); EnumMap credentials = From 2a990457e3e7c15b747b3a3ffbf4487cd69f5b2b Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 6 Sep 2024 12:00:46 -0700 Subject: [PATCH 09/32] more signature fixes --- ...olarisEclipseLinkMetaStoreSessionImpl.java | 9 ++++--- .../polaris/core/catalog/PageToken.java | 2 +- .../BasePolarisMetaStoreManagerTest.java | 3 ++- .../PolarisTestMetaStoreManager.java | 23 +++++++++++----- .../service/admin/PolarisAdminService.java | 13 +++++++--- .../service/catalog/BasePolarisCatalog.java | 26 +++++++++++-------- .../catalog/IcebergCatalogAdapter.java | 11 +++----- .../catalog/PolarisCatalogHandlerWrapper.java | 8 +++--- 8 files changed, 57 insertions(+), 38 deletions(-) diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index d910d0836..a8681a8e7 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -507,8 +507,10 @@ public List lookupEntityActiveBatch( @NotNull PolarisCallContext callCtx, long catalogId, long parentId, - @NotNull PolarisEntityType entityType) { - return listActiveEntities(callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue()); + @NotNull PolarisEntityType entityType, + @NotNull PageToken pageToken) { + return listActiveEntities( + callCtx, catalogId, parentId, entityType, pageToken, Predicates.alwaysTrue()); } @Override @@ -517,6 +519,7 @@ public List lookupEntityActiveBatch( long catalogId, long parentId, @NotNull PolarisEntityType entityType, + @NotNull PageToken pageToken, @NotNull Predicate entityFilter) { // full range scan under the parent for that type return listActiveEntities( @@ -524,7 +527,7 @@ public List lookupEntityActiveBatch( catalogId, parentId, entityType, - PageToken.readEverything(), + pageToken, entityFilter, entity -> new PolarisEntityActiveRecord( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java index 770f3c13b..1ea4ea9a5 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java @@ -68,7 +68,7 @@ public static PageToken fromString(String tokenString) { return new PageToken(offset, pageSize); } catch (Exception e) { - throw new IllegalArgumentException("Failed to decode token: " + tokenString, e); + throw new IllegalArgumentException("Failed to decode page token: " + tokenString, e); } } diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java index 478790bc9..53d2b63df 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java @@ -129,7 +129,8 @@ void testCreateEntities() { polarisTestMetaStoreManager.polarisCallContext, null, PolarisEntityType.TASK, - PolarisEntitySubType.NULL_SUBTYPE) + PolarisEntitySubType.NULL_SUBTYPE, + PageToken.readEverything()) .getEntities(); Assertions.assertThat(listedEntities) .isNotNull() diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java index 893d22927..7dffe6302 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java @@ -28,6 +28,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntity; @@ -653,7 +654,8 @@ void dropEntity(List catalogPath, PolarisEntityCore entityToD this.polarisCallContext, path, PolarisEntityType.NAMESPACE, - PolarisEntitySubType.NULL_SUBTYPE) + PolarisEntitySubType.NULL_SUBTYPE, + PageToken.readEverything()) .getEntities(); Assertions.assertThat(children).isNotNull(); if (children.isEmpty() && entity.getType() == PolarisEntityType.NAMESPACE) { @@ -663,7 +665,8 @@ void dropEntity(List catalogPath, PolarisEntityCore entityToD this.polarisCallContext, path, PolarisEntityType.TABLE_LIKE, - PolarisEntitySubType.ANY_SUBTYPE) + PolarisEntitySubType.ANY_SUBTYPE, + PageToken.readEverything()) .getEntities(); Assertions.assertThat(children).isNotNull(); } else if (children.isEmpty()) { @@ -673,7 +676,8 @@ void dropEntity(List catalogPath, PolarisEntityCore entityToD this.polarisCallContext, path, PolarisEntityType.CATALOG_ROLE, - PolarisEntitySubType.ANY_SUBTYPE) + PolarisEntitySubType.ANY_SUBTYPE, + PageToken.readEverything()) .getEntities(); Assertions.assertThat(children).isNotNull(); // if only one left, it can be dropped. @@ -1182,7 +1186,12 @@ private void validateListReturn( // list the entities under the specified path List result = polarisMetaStoreManager - .listEntities(this.polarisCallContext, path, entityType, entitySubType) + .listEntities( + this.polarisCallContext, + path, + entityType, + entitySubType, + PageToken.readEverything()) .getEntities(); Assertions.assertThat(result).isNotNull(); @@ -1495,7 +1504,8 @@ void validateBootstrap() { this.polarisCallContext, null, PolarisEntityType.PRINCIPAL, - PolarisEntitySubType.NULL_SUBTYPE) + PolarisEntitySubType.NULL_SUBTYPE, + PageToken.readEverything()) .getEntities(); // ensure not null, one element only @@ -1521,7 +1531,8 @@ void validateBootstrap() { this.polarisCallContext, null, PolarisEntityType.PRINCIPAL_ROLE, - PolarisEntitySubType.NULL_SUBTYPE) + PolarisEntitySubType.NULL_SUBTYPE, + PageToken.readEverything()) .getEntities(); // ensure not null, one element only diff --git a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index ebe7d4df6..a8410e6ee 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -56,6 +56,7 @@ import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizableOperation; import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; @@ -721,7 +722,8 @@ private List listCatalogsUnsafe() { getCurrentPolarisContext(), null, PolarisEntityType.CATALOG, - PolarisEntitySubType.ANY_SUBTYPE) + PolarisEntitySubType.ANY_SUBTYPE, + PageToken.readEverything()) .getEntities() .stream() .map( @@ -899,7 +901,8 @@ public List listPrincipals() { getCurrentPolarisContext(), null, PolarisEntityType.PRINCIPAL, - PolarisEntitySubType.NULL_SUBTYPE) + PolarisEntitySubType.NULL_SUBTYPE, + PageToken.readEverything()) .getEntities() .stream() .map( @@ -1021,7 +1024,8 @@ public List listPrincipalRoles() { getCurrentPolarisContext(), null, PolarisEntityType.PRINCIPAL_ROLE, - PolarisEntitySubType.NULL_SUBTYPE) + PolarisEntitySubType.NULL_SUBTYPE, + PageToken.readEverything()) .getEntities() .stream() .map( @@ -1162,7 +1166,8 @@ public List listCatalogRoles(String catalogName) { getCurrentPolarisContext(), PolarisEntity.toCoreList(List.of(catalogEntity)), PolarisEntityType.CATALOG_ROLE, - PolarisEntitySubType.NULL_SUBTYPE) + PolarisEntitySubType.NULL_SUBTYPE, + PageToken.readEverything()) .getEntities() .stream() .map( diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 652669305..54c38970e 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -760,18 +760,21 @@ public PolarisPage listNamespaces(Namespace namespace, PageToken page } List catalogPath = resolvedEntities.getRawFullPath(); - PolarisMetaStoreManager.ListEntitiesResult listResult = entityManager - .getMetaStoreManager() - .listEntities( - getCurrentPolarisContext(), - PolarisEntity.toCoreList(catalogPath), - PolarisEntityType.NAMESPACE, - PolarisEntitySubType.NULL_SUBTYPE, - pageToken); - List entities = PolarisEntity.toNameAndIdList(listResult.getEntities()); + PolarisMetaStoreManager.ListEntitiesResult listResult = + entityManager + .getMetaStoreManager() + .listEntities( + getCurrentPolarisContext(), + PolarisEntity.toCoreList(catalogPath), + PolarisEntityType.NAMESPACE, + PolarisEntitySubType.NULL_SUBTYPE, + pageToken); + List entities = + PolarisEntity.toNameAndIdList(listResult.getEntities()); List namespaces = PolarisCatalogHelpers.nameAndIdToNamespaces(catalogPath, entities); - return listResult.getPageToken() + return listResult + .getPageToken() .map(token -> new PolarisPage<>(token, namespaces)) .orElseGet(() -> PolarisPage.fromData(namespaces)); } @@ -1979,7 +1982,8 @@ private PolarisPage listTableLike( List identifiers = PolarisCatalogHelpers.nameAndIdToTableIdentifiers(catalogPath, entities); - return listResult.getPageToken() + return listResult + .getPageToken() .map(token -> new PolarisPage<>(token, identifiers)) .orElseGet(() -> PolarisPage.fromData(identifiers)); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 7dec28997..b0e94ce0c 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -50,7 +50,6 @@ import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.catalog.PageToken; -import org.apache.polaris.core.catalog.PolarisPage; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisEntity; @@ -64,7 +63,6 @@ import org.apache.polaris.service.context.CallContextCatalogFactory; import org.apache.polaris.service.types.CommitTableRequest; import org.apache.polaris.service.types.CommitViewRequest; -import org.apache.polaris.service.types.ListTablesResponseWithPageToken; import org.apache.polaris.service.types.NotificationRequest; /** @@ -131,8 +129,8 @@ public Response listNamespaces( PageToken token = PageToken.fromString(pageToken).withPageSize(pageSize); return Response.ok( - newHandlerWrapper(securityContext, prefix) - .listNamespaces(namespaceOptional.orElse(Namespace.of()), token)) + newHandlerWrapper(securityContext, prefix) + .listNamespaces(namespaceOptional.orElse(Namespace.of()), token)) .build(); } @@ -220,10 +218,7 @@ public Response listTables( PageToken token = PageToken.fromString(pageToken).withPageSize(pageSize); - return Response.ok( - newHandlerWrapper(securityContext, prefix) - .listTables(ns, token)) - .build(); + return Response.ok(newHandlerWrapper(securityContext, prefix).listTables(ns, token)).build(); } @Override diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java index 9e2724a4b..8ca444c51 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java @@ -418,11 +418,11 @@ public ListNamespacesResponseWithPageToken listNamespaces(Namespace parent, Page authorizeBasicNamespaceOperationOrThrow(op, parent); if (baseCatalog instanceof BasePolarisCatalog bpc) { - ListNamespacesResponseWithPageToken - .fromPolarisPage(doCatalogOperation(() -> bpc.listNamespaces(parent, pageToken))); + return ListNamespacesResponseWithPageToken.fromPolarisPage( + doCatalogOperation(() -> bpc.listNamespaces(parent, pageToken))); } else { - return ListNamespacesResponseWithPageToken - .fromPolarisPage(PolarisPage.fromData( + return ListNamespacesResponseWithPageToken.fromPolarisPage( + PolarisPage.fromData( doCatalogOperation(() -> CatalogHandlers.listNamespaces(namespaceCatalog, parent)) .namespaces())); } From 14b0ef37d50ee6da855541c2eabbd4044bb024a6 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 6 Sep 2024 12:48:03 -0700 Subject: [PATCH 10/32] tests maybe working --- .../polaris/service/catalog/IcebergCatalogAdapter.java | 8 ++++---- .../types/ListNamespacesResponseWithPageToken.java | 3 ++- .../service/types/ListTablesResponseWithPageToken.java | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index b0e94ce0c..5147e496d 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -128,10 +128,10 @@ public Response listNamespaces( Optional.ofNullable(parent).map(IcebergCatalogAdapter::decodeNamespace); PageToken token = PageToken.fromString(pageToken).withPageSize(pageSize); - return Response.ok( - newHandlerWrapper(securityContext, prefix) - .listNamespaces(namespaceOptional.orElse(Namespace.of()), token)) - .build(); + var response = + newHandlerWrapper(securityContext, prefix) + .listNamespaces(namespaceOptional.orElse(Namespace.of()), token); + return Response.ok(response).build(); } @Override diff --git a/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java b/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java index b5c0391bf..9ef1515e6 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; import java.util.List; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.rest.responses.ListNamespacesResponse; @@ -35,7 +36,7 @@ public class ListNamespacesResponseWithPageToken extends ListNamespacesResponse public ListNamespacesResponseWithPageToken(PageToken pageToken, List namespaces) { this.pageToken = pageToken; this.namespaces = namespaces; - this.validate(); + Preconditions.checkArgument(this.namespaces != null, "Invalid namespace: null"); } public static ListNamespacesResponseWithPageToken fromPolarisPage( diff --git a/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java b/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java index d073ea202..1aa725027 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; import java.util.List; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.rest.responses.ListTablesResponse; @@ -35,7 +36,7 @@ public class ListTablesResponseWithPageToken extends ListTablesResponse { public ListTablesResponseWithPageToken(PageToken pageToken, List identifiers) { this.pageToken = pageToken; this.identifiers = identifiers; - this.validate(); + Preconditions.checkArgument(this.identifiers != null, "Invalid identifier list: null"); } public static ListTablesResponseWithPageToken fromPolarisPage( From 8339e520a968121982e0fce41ad13b2bd252d031 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 6 Sep 2024 13:32:45 -0700 Subject: [PATCH 11/32] tested --- .../apache/polaris/core/catalog/PageToken.java | 15 +++++++++++---- .../core/persistence/PolarisMetaStoreManager.java | 4 ++-- .../persistence/PolarisMetaStoreManagerImpl.java | 5 +++-- .../PolarisTreeMapMetaStoreSessionImpl.java | 2 +- .../service/catalog/IcebergCatalogAdapter.java | 1 - .../ListNamespacesResponseWithPageToken.java | 12 ++++++++---- .../types/ListTablesResponseWithPageToken.java | 10 +++++++--- 7 files changed, 32 insertions(+), 17 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java index 1ea4ea9a5..160f97832 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java @@ -52,7 +52,7 @@ public static PageToken readEverything() { /** Decode a token string into a PaginationToken object */ public static PageToken fromString(String tokenString) { if (tokenString == null || tokenString.isEmpty()) { - return new PageToken(TOKEN_START, DEFAULT_PAGE_SIZE); + return PageToken.readEverything(); } try { @@ -72,16 +72,23 @@ public static PageToken fromString(String tokenString) { } } - /** Builds a new page token to reflect new data that's been read */ + /** + * Builds a new page token to reflect new data that's been read. If the amount of data read is + * less than the pageSize, this will return `PageToken.DONE` (done) + */ public PageToken updated(List newData) { - return new PageToken(offset + newData.size(), pageSize); + if (newData == null || newData.isEmpty() || newData.size() < pageSize) { + return PageToken.DONE; + } else { + return new PageToken(offset + newData.size(), pageSize); + } } public PageToken withPageSize(Integer pageSize) { if (pageSize == null) { return this; } else { - return new PageToken(offset, pageSize); + return new PageToken(this.offset, pageSize); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java index f7c528ea8..3041be341 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java @@ -316,7 +316,7 @@ class ListEntitiesResult extends BaseResult { public static ListEntitiesResult fromPolarisPage( PolarisPage polarisPage) { - return new ListEntitiesResult(polarisPage.data, Optional.of(polarisPage.pageToken)); + return new ListEntitiesResult(polarisPage.data, Optional.ofNullable(polarisPage.pageToken)); } /** @@ -687,7 +687,7 @@ class EntitiesResult extends BaseResult { private final Optional pageTokenOpt; public static EntitiesResult fromPolarisPage(PolarisPage polarisPage) { - return new EntitiesResult(polarisPage.data, Optional.of(polarisPage.pageToken)); + return new EntitiesResult(polarisPage.data, Optional.ofNullable(polarisPage.pageToken)); } /** diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 2f6d0206e..78dde52a7 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -781,7 +781,8 @@ private void bootstrapPolarisService( } /** - * See {@link #listEntities(PolarisCallContext, List, PolarisEntityType, PolarisEntitySubType)} + * See {@link #listEntities(PolarisCallContext, List, PolarisEntityType, PolarisEntitySubType, + * PageToken)} */ private @NotNull ListEntitiesResult listEntities( @NotNull PolarisCallContext callCtx, @@ -808,7 +809,7 @@ private void bootstrapPolarisService( if (entitySubType != PolarisEntitySubType.ANY_SUBTYPE) { resultPage = new PolarisPage( - resultPage.pageToken, + pageToken.updated(resultPage.data), resultPage.data.stream() .filter(rec -> rec.getSubTypeCode() == entitySubType.getCode()) .collect(Collectors.toList())); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java index b4dfda055..a6d77e50b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java @@ -345,7 +345,7 @@ public List lookupEntityActiveBatch( catalogId, parentId, entityType, - PageToken.readEverything(), + pageToken, entityFilter, entity -> new PolarisEntityActiveRecord( diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 5147e496d..856bc8954 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -217,7 +217,6 @@ public Response listTables( Namespace ns = decodeNamespace(namespace); PageToken token = PageToken.fromString(pageToken).withPageSize(pageSize); - return Response.ok(newHandlerWrapper(securityContext, prefix).listTables(ns, token)).build(); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java b/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java index 9ef1515e6..f9e9b412b 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java @@ -28,7 +28,6 @@ import org.apache.polaris.core.catalog.PolarisPage; public class ListNamespacesResponseWithPageToken extends ListNamespacesResponse { - @JsonProperty("next-page-token") private final PageToken pageToken; private final List namespaces; @@ -44,8 +43,13 @@ public static ListNamespacesResponseWithPageToken fromPolarisPage( return new ListNamespacesResponseWithPageToken(polarisPage.pageToken, polarisPage.data); } - public PageToken getPageToken() { - return pageToken; + @JsonProperty("next-page-token") + public String getPageToken() { + if (pageToken == null) { + return null; + } else { + return pageToken.toString(); + } } @Override @@ -57,7 +61,7 @@ public List namespaces() { public String toString() { return MoreObjects.toStringHelper(this) .add("namespaces", this.namespaces) - .add("pageToken", this.pageToken) + .add("pageToken", this.pageToken.toString()) .toString(); } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java b/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java index 1aa725027..662936f45 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java @@ -28,7 +28,6 @@ import org.apache.polaris.core.catalog.PolarisPage; public class ListTablesResponseWithPageToken extends ListTablesResponse { - @JsonProperty("next-page-token") private final PageToken pageToken; private final List identifiers; @@ -44,8 +43,13 @@ public static ListTablesResponseWithPageToken fromPolarisPage( return new ListTablesResponseWithPageToken(polarisPage.pageToken, polarisPage.data); } - public PageToken getPageToken() { - return pageToken; + @JsonProperty("next-page-token") + public String getPageToken() { + if (pageToken == null) { + return null; + } else { + return pageToken.toString(); + } } @Override From dcaf7983e80a211ee65c222dbad7ec1d20026a24 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 6 Sep 2024 13:47:44 -0700 Subject: [PATCH 12/32] add config, fixes --- .../polaris/core/PolarisConfiguration.java | 7 ++++++ .../polaris/core/catalog/PageToken.java | 23 +++++++++++++++---- .../service/catalog/BasePolarisCatalog.java | 23 ++++++++++++++++--- .../catalog/IcebergCatalogAdapter.java | 3 ++- .../catalog/PolarisCatalogHandlerWrapper.java | 15 ++++++++++++ 5 files changed, 62 insertions(+), 9 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java index 31af621d8..3c902a704 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java @@ -150,4 +150,11 @@ public static Builder builder() { "If set to true, allows tables to have external locations outside the default structure.") .defaultValue(false) .build(); + + public static final PolarisConfiguration PAGINATION_ENABLED = + PolarisConfiguration.builder() + .key("PAGINATION_ENABLED") + .description("If set to true, pagination for APIs like listTables is enabled") + .defaultValue(true) + .build(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java index 160f97832..13e10952b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java @@ -22,7 +22,15 @@ import java.util.Base64; import java.util.List; -/** Represents a page token */ +/** + * Represents a page token that can be used by operations like `listTables`. Clients that specify a + * `pageSize` (or a `pageToken`) may receive a `next-page-token` in the response, the content of + * which is a serialized PageToken. + * + *

By providing that in the next query's `pageToken`, the client can resume listing where they + * left off. If the client provides a `pageToken` or `pageSize` but `next-page-token` is null in the + * response, that means there is no more data to read. + */ public class PageToken { public final int offset; @@ -39,17 +47,17 @@ public PageToken(int offset, int pageSize) { this.pageSize = pageSize; } - /** Construct a PaginationToken from a plain limit */ + /** Construct a PageToken from a plain limit */ public static PageToken fromLimit(int limit) { return new PageToken(TOKEN_START, limit); } - /** Construct a PaginationToken from a plain limit */ + /** Construct a PageToken to read everything */ public static PageToken readEverything() { return new PageToken(TOKEN_START, Integer.MAX_VALUE); } - /** Decode a token string into a PaginationToken object */ + /** Deserialize a token string into a PageToken object */ public static PageToken fromString(String tokenString) { if (tokenString == null || tokenString.isEmpty()) { return PageToken.readEverything(); @@ -84,14 +92,19 @@ public PageToken updated(List newData) { } } + /** + * Return a new PageToken with an updated pageSize. If the pageSize provided is null, the existing + * pageSize will be preserved. + */ public PageToken withPageSize(Integer pageSize) { if (pageSize == null) { - return this; + return new PageToken(this.offset, this.pageSize); } else { return new PageToken(this.offset, pageSize); } } + /** Serialize a PageToken into a string */ @Override public String toString() { String tokenContent = TOKEN_PREFIX + ":" + offset + ":" + pageSize; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 54c38970e..235b8d851 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -449,6 +449,14 @@ public boolean dropTable(TableIdentifier tableIdentifier, boolean purge) { return true; } + private boolean paginationEnabled() { + return callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration( + callContext.getPolarisCallContext(), PolarisConfiguration.PAGINATION_ENABLED); + } + @Override public List listTables(Namespace namespace) { return listTables(namespace, PageToken.readEverything()).data; @@ -459,6 +467,9 @@ public PolarisPage listTables(Namespace namespace, PageToken pa throw new NoSuchNamespaceException( "Cannot list tables for namespace. Namespace does not exist: %s", namespace); } + if (!paginationEnabled()) { + return PolarisPage.fromData(listViews(namespace)); + } return listTableLike(PolarisEntitySubType.TABLE, namespace, pageToken); } @@ -758,6 +769,9 @@ public PolarisPage listNamespaces(Namespace namespace, PageToken page if (resolvedEntities == null) { throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); } + if (!paginationEnabled()) { + return PolarisPage.fromData(listNamespaces(namespace)); + } List catalogPath = resolvedEntities.getRawFullPath(); PolarisMetaStoreManager.ListEntitiesResult listResult = @@ -784,16 +798,19 @@ public void close() throws IOException {} @Override public List listViews(Namespace namespace) { - return listViews(namespace, PageToken.readEverything()); + return listViews(namespace, PageToken.readEverything()).data; } - public List listViews(Namespace namespace, PageToken pageToken) { + public PolarisPage listViews(Namespace namespace, PageToken pageToken) { if (!namespaceExists(namespace) && !namespace.isEmpty()) { throw new NoSuchNamespaceException( "Cannot list views for namespace. Namespace does not exist: %s", namespace); } + if (!paginationEnabled()) { + return PolarisPage.fromData(listViews(namespace)); + } - return listTableLike(PolarisEntitySubType.VIEW, namespace, pageToken).data; + return listTableLike(PolarisEntitySubType.VIEW, namespace, pageToken); } @Override diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 856bc8954..76e58958f 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -352,7 +352,8 @@ public Response listViews( Integer pageSize, SecurityContext securityContext) { Namespace ns = decodeNamespace(namespace); - return Response.ok(newHandlerWrapper(securityContext, prefix).listViews(ns)).build(); + PageToken token = PageToken.fromString(pageToken).withPageSize(pageSize); + return Response.ok(newHandlerWrapper(securityContext, prefix).listViews(ns, token)).build(); } @Override diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java index 8ca444c51..002457257 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java @@ -1016,6 +1016,21 @@ public ListTablesResponse listViews(Namespace namespace) { return doCatalogOperation(() -> CatalogHandlers.listViews(viewCatalog, namespace)); } + public ListTablesResponseWithPageToken listViews(Namespace namespace, PageToken pageToken) { + PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_VIEWS; + authorizeBasicNamespaceOperationOrThrow(op, namespace); + + if (baseCatalog instanceof BasePolarisCatalog bpc) { + return ListTablesResponseWithPageToken.fromPolarisPage( + doCatalogOperation(() -> bpc.listViews(namespace, pageToken))); + } else { + return ListTablesResponseWithPageToken.fromPolarisPage( + PolarisPage.fromData( + doCatalogOperation(() -> CatalogHandlers.listTables(baseCatalog, namespace)) + .identifiers())); + } + } + public LoadViewResponse createView(Namespace namespace, CreateViewRequest request) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_VIEW; authorizeCreateTableLikeUnderNamespaceOperationOrThrow( From 6b1c36207ebf2c59f3cf2b99c92288b883498db0 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 6 Sep 2024 14:01:35 -0700 Subject: [PATCH 13/32] improvements --- ...olarisEclipseLinkMetaStoreSessionImpl.java | 2 +- .../core/PolarisConfigurationStore.java | 2 + .../polaris/core/catalog/PageToken.java | 39 ++++++++++++------- .../polaris/core/catalog/PolarisPage.java | 4 ++ .../PolarisMetaStoreManagerImpl.java | 3 +- .../ListNamespacesResponseWithPageToken.java | 5 +++ 6 files changed, 38 insertions(+), 17 deletions(-) diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index a8681a8e7..d6b8abf2b 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -558,7 +558,7 @@ public List lookupEntityActiveBatch( .limit(pageToken.pageSize) .map(transformer) .collect(Collectors.toList()); - return new PolarisPage(pageToken.updated(data), data); + return pageToken.buildNextPage(data); } /** {@inheritDoc} */ diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfigurationStore.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfigurationStore.java index a249ed907..95c926886 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfigurationStore.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfigurationStore.java @@ -68,6 +68,8 @@ public interface PolarisConfigurationStore { if (config.defaultValue instanceof Boolean) { return config.cast(Boolean.valueOf(String.valueOf(value))); + } else if (config.defaultValue instanceof Integer) { + return config.cast(Integer.valueOf(value.toString())); } else { return config.cast(value); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java index 13e10952b..0b2d63cae 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java @@ -59,24 +59,27 @@ public static PageToken readEverything() { /** Deserialize a token string into a PageToken object */ public static PageToken fromString(String tokenString) { - if (tokenString == null || tokenString.isEmpty()) { + if (tokenString == null) { return PageToken.readEverything(); - } + } else if (tokenString.isEmpty()) { + return PageToken.fromLimit(DEFAULT_PAGE_SIZE); + } else { + try { + String decoded = + new String(Base64.getDecoder().decode(tokenString), StandardCharsets.UTF_8); + String[] parts = decoded.split(":"); - try { - String decoded = new String(Base64.getDecoder().decode(tokenString), StandardCharsets.UTF_8); - String[] parts = decoded.split(":"); + if (parts.length != 3 || !parts[0].equals(TOKEN_PREFIX)) { + throw new IllegalArgumentException("Invalid token format"); + } - if (parts.length != 3 || !parts[0].equals(TOKEN_PREFIX)) { - throw new IllegalArgumentException("Invalid token format"); - } + int offset = Integer.parseInt(parts[1]); + int pageSize = Integer.parseInt(parts[2]); - int offset = Integer.parseInt(parts[1]); - int pageSize = Integer.parseInt(parts[2]); - - return new PageToken(offset, pageSize); - } catch (Exception e) { - throw new IllegalArgumentException("Failed to decode page token: " + tokenString, e); + return new PageToken(offset, pageSize); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to decode page token: " + tokenString, e); + } } } @@ -92,6 +95,14 @@ public PageToken updated(List newData) { } } + /** + * Builds a `PolarisPage` from a `List`. The `PageToken` attached to the new + * `PolarisPage` is the same as the result of calling `updated(data)` on this `PageToken`. + */ + public PolarisPage buildNextPage(List data) { + return new PolarisPage(this.updated(data), data); + } + /** * Return a new PageToken with an updated pageSize. If the pageSize provided is null, the existing * pageSize will be preserved. diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PolarisPage.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PolarisPage.java index ba58cf5fb..c8382e024 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PolarisPage.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PolarisPage.java @@ -20,6 +20,10 @@ import java.util.List; +/** + * A wrapper for a List of data and a PageToken that can be used to continue the listing operation + * that generated that data. + */ public class PolarisPage { public final PageToken pageToken; public final List data; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 78dde52a7..1bb865542 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -808,8 +808,7 @@ private void bootstrapPolarisService( // prune the returned list with only entities matching the entity subtype if (entitySubType != PolarisEntitySubType.ANY_SUBTYPE) { resultPage = - new PolarisPage( - pageToken.updated(resultPage.data), + pageToken.buildNextPage( resultPage.data.stream() .filter(rec -> rec.getSubTypeCode() == entitySubType.getCode()) .collect(Collectors.toList())); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java b/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java index f9e9b412b..a35323ae3 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java @@ -27,6 +27,11 @@ import org.apache.polaris.core.catalog.PageToken; import org.apache.polaris.core.catalog.PolarisPage; +/** + * Used in lieu of `ListNamespacesResponse` when there may be a `PageToken` associated with the + * response. Callers can use this `PageToken` to continue the listing operation and obtain more + * results. + */ public class ListNamespacesResponseWithPageToken extends ListNamespacesResponse { private final PageToken pageToken; From ccd0713cb0bd2d7fedba1b314a5e0a1bbe49da40 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 6 Sep 2024 14:07:22 -0700 Subject: [PATCH 14/32] implement checksum --- .../org/apache/polaris/core/catalog/PageToken.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java index 0b2d63cae..e49189454 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java @@ -69,14 +69,20 @@ public static PageToken fromString(String tokenString) { new String(Base64.getDecoder().decode(tokenString), StandardCharsets.UTF_8); String[] parts = decoded.split(":"); - if (parts.length != 3 || !parts[0].equals(TOKEN_PREFIX)) { - throw new IllegalArgumentException("Invalid token format"); + if (parts.length != 4 || !parts[0].equals(TOKEN_PREFIX)) { + throw new IllegalArgumentException("Invalid token format in token: " + tokenString); } int offset = Integer.parseInt(parts[1]); int pageSize = Integer.parseInt(parts[2]); + int checksum = Integer.parseInt(parts[3]); + PageToken token = new PageToken(offset, pageSize); - return new PageToken(offset, pageSize); + if (token.hashCode() != checksum) { + throw new IllegalArgumentException("Invalid checksum for token: " + tokenString); + } else { + return token; + } } catch (Exception e) { throw new IllegalArgumentException("Failed to decode page token: " + tokenString, e); } @@ -118,7 +124,7 @@ public PageToken withPageSize(Integer pageSize) { /** Serialize a PageToken into a string */ @Override public String toString() { - String tokenContent = TOKEN_PREFIX + ":" + offset + ":" + pageSize; + String tokenContent = TOKEN_PREFIX + ":" + offset + ":" + pageSize + ":" + hashCode(); return Base64.getEncoder().encodeToString(tokenContent.getBytes(StandardCharsets.UTF_8)); } From feb8c29ad4c2e662c2d77155e801dd46b9a299d6 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 6 Sep 2024 14:26:38 -0700 Subject: [PATCH 15/32] more tests --- .../service/catalog/BasePolarisCatalog.java | 4 + .../catalog/BasePolarisCatalogTest.java | 102 ++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 235b8d851..72b41c806 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -763,6 +763,10 @@ public List listNamespaces(Namespace namespace) throws NoSuchNamespac return listNamespaces(namespace, PageToken.readEverything()).data; } + public PolarisPage listNamespaces(PageToken pageToken) throws NoSuchNamespaceException { + return listNamespaces(Namespace.empty(), pageToken); + } + public PolarisPage listNamespaces(Namespace namespace, PageToken pageToken) throws NoSuchNamespaceException { PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(namespace); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index 12371d31c..63785f0e0 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -63,6 +63,7 @@ import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.catalog.PageToken; +import org.apache.polaris.core.catalog.PolarisPage; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.CatalogEntity; @@ -1243,4 +1244,105 @@ public void testFileIOWrapper() { .getFirst())); Assertions.assertThat(measured.getNumDeletedFiles()).as("A table was deleted").isGreaterThan(0); } + + @Test + public void testPaginatedListTables() { + if (this.requiresNamespaceCreate()) { + ((SupportsNamespaces) catalog).createNamespace(NS); + } + + for (int i = 0; i < 5; i++) { + catalog.buildTable(TableIdentifier.of(NS, "pagination_table_" + i), SCHEMA).create(); + } + + try { + // List without pagination + Assertions.assertThat(catalog.listTables(NS)).isNotNull().hasSize(5); + + // List with a limit: + PolarisPage firstListResult = catalog.listTables(NS, PageToken.fromLimit(2)); + Assertions.assertThat(firstListResult.data.size()).isEqualTo(2); + Assertions.assertThat(firstListResult.pageToken.toString()).isNotNull().isNotEmpty(); + + // List using the previously obtained token: + PolarisPage secondListResult = catalog.listTables(NS, firstListResult.pageToken); + Assertions.assertThat(secondListResult.data.size()).isEqualTo(2); + Assertions.assertThat(secondListResult.pageToken.toString()).isNotNull().isNotEmpty(); + + // List using the final token: + PolarisPage finalListResult = catalog.listTables(NS, secondListResult.pageToken); + Assertions.assertThat(finalListResult.data.size()).isEqualTo(1); + Assertions.assertThat(finalListResult.pageToken).isNull(); + } finally { + for (int i = 0; i < 5; i++) { + catalog.dropTable(TableIdentifier.of(NS, "pagination_table_" + i)); + } + } + } + + @Test + public void testPaginatedListViews() { + if (this.requiresNamespaceCreate()) { + ((SupportsNamespaces) catalog).createNamespace(NS); + } + + for (int i = 0; i < 5; i++) { + catalog.buildView(TableIdentifier.of(NS, "pagination_view_" + i)).create(); + } + + try { + // List without pagination + Assertions.assertThat(catalog.listViews(NS)).isNotNull().hasSize(5); + + // List with a limit: + PolarisPage firstListResult = catalog.listViews(NS, PageToken.fromLimit(2)); + Assertions.assertThat(firstListResult.data.size()).isEqualTo(2); + Assertions.assertThat(firstListResult.pageToken.toString()).isNotNull().isNotEmpty(); + + // List using the previously obtained token: + PolarisPage secondListResult = catalog.listViews(NS, firstListResult.pageToken); + Assertions.assertThat(secondListResult.data.size()).isEqualTo(2); + Assertions.assertThat(secondListResult.pageToken.toString()).isNotNull().isNotEmpty(); + + // List using the final token: + PolarisPage finalListResult = catalog.listViews(NS, secondListResult.pageToken); + Assertions.assertThat(finalListResult.data.size()).isEqualTo(1); + Assertions.assertThat(finalListResult.pageToken).isNull(); + } finally { + for (int i = 0; i < 5; i++) { + catalog.dropTable(TableIdentifier.of(NS, "pagination_view_" + i)); + } + } + } + + @Test + public void testPaginatedListNamespaces() { + for (int i = 0; i < 5; i++) { + catalog.createNamespace(Namespace.of("pagination_namespace_" + i)); + } + + try { + // List without pagination + Assertions.assertThat(catalog.listNamespaces()).isNotNull().hasSize(5); + + // List with a limit: + PolarisPage firstListResult = catalog.listNamespaces(PageToken.fromLimit(2)); + Assertions.assertThat(firstListResult.data.size()).isEqualTo(2); + Assertions.assertThat(firstListResult.pageToken.toString()).isNotNull().isNotEmpty(); + + // List using the previously obtained token: + PolarisPage secondListResult = catalog.listNamespaces(firstListResult.pageToken); + Assertions.assertThat(secondListResult.data.size()).isEqualTo(2); + Assertions.assertThat(secondListResult.pageToken.toString()).isNotNull().isNotEmpty(); + + // List using the final token: + PolarisPage finalListResult = catalog.listNamespaces(secondListResult.pageToken); + Assertions.assertThat(finalListResult.data.size()).isEqualTo(1); + Assertions.assertThat(finalListResult.pageToken).isNull(); + } finally { + for (int i = 0; i < 5; i++) { + catalog.dropNamespace(Namespace.of("pagination_namespace_" + i)); + } + } + } } From d649725d5df71084c9963069bf3e11649305db20 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Sun, 8 Sep 2024 21:49:13 -0400 Subject: [PATCH 16/32] one tweak --- .../main/java/org/apache/polaris/core/catalog/PageToken.java | 5 +++++ .../apache/polaris/service/catalog/BasePolarisCatalog.java | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java index e49189454..ce22785fc 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java @@ -43,6 +43,11 @@ public class PageToken { public static PageToken DONE = null; public PageToken(int offset, int pageSize) { + if (offset < 0 || pageSize <= 0) { + throw new IllegalArgumentException( + "Invalid token content (offset / pageSize): " + offset + " / " + pageSize); + } + this.offset = offset; this.pageSize = pageSize; } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 72b41c806..196a6e60e 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -763,7 +763,8 @@ public List listNamespaces(Namespace namespace) throws NoSuchNamespac return listNamespaces(namespace, PageToken.readEverything()).data; } - public PolarisPage listNamespaces(PageToken pageToken) throws NoSuchNamespaceException { + public PolarisPage listNamespaces(PageToken pageToken) + throws NoSuchNamespaceException { return listNamespaces(Namespace.empty(), pageToken); } From c14797f9f596e64c6c6ae3665d3abcfb18fa2ee8 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 9 Sep 2024 10:25:40 -0400 Subject: [PATCH 17/32] add another test case --- .../service/catalog/BasePolarisCatalogTest.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index 63785f0e0..22dd8b701 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -1339,6 +1339,16 @@ public void testPaginatedListNamespaces() { PolarisPage finalListResult = catalog.listNamespaces(secondListResult.pageToken); Assertions.assertThat(finalListResult.data.size()).isEqualTo(1); Assertions.assertThat(finalListResult.pageToken).isNull(); + + // List with large page size + PolarisPage exactListResult = catalog.listNamespaces(PageToken.fromLimit(5)); + Assertions.assertThat(exactListResult.data.size()).isEqualTo(5); + Assertions.assertThat(exactListResult.pageToken).isNull(); + + // List with huge page size: + PolarisPage bigListResult = catalog.listNamespaces(PageToken.fromLimit(9999)); + Assertions.assertThat(bigListResult.data.size()).isEqualTo(5); + Assertions.assertThat(bigListResult.pageToken).isNull(); } finally { for (int i = 0; i < 5; i++) { catalog.dropNamespace(Namespace.of("pagination_namespace_" + i)); From 840307603343b803900e77689a846850983aebad Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 9 Sep 2024 10:25:40 -0400 Subject: [PATCH 18/32] add another test case fixes --- .../service/catalog/BasePolarisCatalogTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index 63785f0e0..2cc8a0e40 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -1339,6 +1339,21 @@ public void testPaginatedListNamespaces() { PolarisPage finalListResult = catalog.listNamespaces(secondListResult.pageToken); Assertions.assertThat(finalListResult.data.size()).isEqualTo(1); Assertions.assertThat(finalListResult.pageToken).isNull(); + + // List with page size matching the amount of data + PolarisPage firstExactListResult = catalog.listNamespaces(PageToken.fromLimit(5)); + Assertions.assertThat(firstExactListResult.data.size()).isEqualTo(5); + Assertions.assertThat(firstExactListResult.pageToken.toString()).isNotNull().isNotEmpty(); + + // Again list with matching page size + PolarisPage secondExactListResult = catalog.listNamespaces(firstExactListResult.pageToken); + Assertions.assertThat(secondExactListResult.data).isEmpty(); + Assertions.assertThat(secondExactListResult.pageToken).isNull(); + + // List with huge page size: + PolarisPage bigListResult = catalog.listNamespaces(PageToken.fromLimit(9999)); + Assertions.assertThat(bigListResult.data.size()).isEqualTo(5); + Assertions.assertThat(bigListResult.pageToken).isNull(); } finally { for (int i = 0; i < 5; i++) { catalog.dropNamespace(Namespace.of("pagination_namespace_" + i)); From fe0acc802a79a1e1eada75b792a59cd8fe8c3eb8 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 9 Sep 2024 13:12:08 -0400 Subject: [PATCH 19/32] check in --- ...olarisEclipseLinkMetaStoreSessionImpl.java | 16 +- .../eclipselink/PolarisEclipseLinkStore.java | 20 ++- .../polaris/core/catalog/PageToken.java | 150 ------------------ .../catalog/pagination/EntityIdPageToken.java | 74 +++++++++ .../catalog/pagination/OffsetPageToken.java | 99 ++++++++++++ .../core/catalog/pagination/PageToken.java | 147 +++++++++++++++++ .../catalog/{ => pagination}/PolarisPage.java | 2 +- .../persistence/PolarisMetaStoreManager.java | 4 +- .../PolarisMetaStoreManagerImpl.java | 9 +- .../persistence/PolarisMetaStoreSession.java | 13 +- .../PolarisTreeMapMetaStoreSessionImpl.java | 21 ++- .../BasePolarisMetaStoreManagerTest.java | 2 +- .../PolarisTestMetaStoreManager.java | 2 +- .../service/admin/PolarisAdminService.java | 10 +- .../service/catalog/BasePolarisCatalog.java | 16 +- .../catalog/IcebergCatalogAdapter.java | 20 ++- .../catalog/PolarisCatalogHandlerWrapper.java | 4 +- .../ListNamespacesResponseWithPageToken.java | 4 +- .../ListTablesResponseWithPageToken.java | 4 +- .../catalog/BasePolarisCatalogTest.java | 31 ++-- .../task/TableCleanupTaskHandlerTest.java | 23 ++- 21 files changed, 457 insertions(+), 214 deletions(-) delete mode 100644 polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java rename polaris-core/src/main/java/org/apache/polaris/core/catalog/{ => pagination}/PolarisPage.java (96%) diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index d6b8abf2b..bf6c4dd57 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -48,8 +48,9 @@ import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.catalog.PageToken; -import org.apache.polaris.core.catalog.PolarisPage; +import org.apache.polaris.core.catalog.pagination.EntityIdPageToken; +import org.apache.polaris.core.catalog.pagination.PageToken; +import org.apache.polaris.core.catalog.pagination.PolarisPage; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; @@ -550,12 +551,10 @@ public List lookupEntityActiveBatch( @NotNull Function transformer) { List data = this.store - .lookupFullEntitiesActive(localSession.get(), catalogId, parentId, entityType) - .stream() + .lookupFullEntitiesActive( + localSession.get(), catalogId, parentId, entityType, pageToken) .map(ModelEntity::toEntity) .filter(entityFilter) - .skip(pageToken.offset) - .limit(pageToken.pageSize) .map(transformer) .collect(Collectors.toList()); return pageToken.buildNextPage(data); @@ -768,4 +767,9 @@ public void rollback() { session.getTransaction().rollback(); } } + + @Override + public @NotNull PageToken.PageTokenBuilder pageTokenBuilder() { + return new EntityIdPageToken.EntityIdPageTokenBuilder(); + } } diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java index 02461fa4f..ede128412 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java @@ -23,7 +23,10 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.catalog.pagination.EntityIdPageToken; +import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; import org.apache.polaris.core.entity.PolarisEntityActiveRecord; @@ -273,22 +276,29 @@ long countActiveChildEntities( return query.getSingleResult(); } - List lookupFullEntitiesActive( - EntityManager session, long catalogId, long parentId, @NotNull PolarisEntityType entityType) { + Stream lookupFullEntitiesActive( + EntityManager session, + long catalogId, + long parentId, + @NotNull PolarisEntityType entityType, + @NotNull PageToken pageToken) { diagnosticServices.check(session != null, "session_is_null"); + diagnosticServices.check(pageToken instanceof EntityIdPageToken, "unexpected_page_token"); // Currently check against ENTITIES not joining with ENTITIES_ACTIVE String hql = - "SELECT m from ModelEntity m where m.catalogId=:catalogId and m.parentId=:parentId and m.typeCode=:typeCode"; + "SELECT m from ModelEntity m where m.catalogId=:catalogId and m.parentId=:parentId and m.typeCode=:typeCode and m.id > :tokenId"; TypedQuery query = session .createQuery(hql, ModelEntity.class) .setParameter("catalogId", catalogId) .setParameter("parentId", parentId) - .setParameter("typeCode", entityType.getCode()); + .setParameter("typeCode", entityType.getCode()) + .setParameter("tokenId", ((EntityIdPageToken) pageToken).id) + .setMaxResults(pageToken.pageSize); - return query.getResultList(); + return query.getResultStream(); } ModelEntityDropped lookupEntityDropped(EntityManager session, long catalogId, long entityId) { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java deleted file mode 100644 index ce22785fc..000000000 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PageToken.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.core.catalog; - -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.List; - -/** - * Represents a page token that can be used by operations like `listTables`. Clients that specify a - * `pageSize` (or a `pageToken`) may receive a `next-page-token` in the response, the content of - * which is a serialized PageToken. - * - *

By providing that in the next query's `pageToken`, the client can resume listing where they - * left off. If the client provides a `pageToken` or `pageSize` but `next-page-token` is null in the - * response, that means there is no more data to read. - */ -public class PageToken { - - public final int offset; - public final int pageSize; - - private static final String TOKEN_PREFIX = "polaris"; - private static final int TOKEN_START = 0; - private static final int DEFAULT_PAGE_SIZE = 1000; - - public static PageToken DONE = null; - - public PageToken(int offset, int pageSize) { - if (offset < 0 || pageSize <= 0) { - throw new IllegalArgumentException( - "Invalid token content (offset / pageSize): " + offset + " / " + pageSize); - } - - this.offset = offset; - this.pageSize = pageSize; - } - - /** Construct a PageToken from a plain limit */ - public static PageToken fromLimit(int limit) { - return new PageToken(TOKEN_START, limit); - } - - /** Construct a PageToken to read everything */ - public static PageToken readEverything() { - return new PageToken(TOKEN_START, Integer.MAX_VALUE); - } - - /** Deserialize a token string into a PageToken object */ - public static PageToken fromString(String tokenString) { - if (tokenString == null) { - return PageToken.readEverything(); - } else if (tokenString.isEmpty()) { - return PageToken.fromLimit(DEFAULT_PAGE_SIZE); - } else { - try { - String decoded = - new String(Base64.getDecoder().decode(tokenString), StandardCharsets.UTF_8); - String[] parts = decoded.split(":"); - - if (parts.length != 4 || !parts[0].equals(TOKEN_PREFIX)) { - throw new IllegalArgumentException("Invalid token format in token: " + tokenString); - } - - int offset = Integer.parseInt(parts[1]); - int pageSize = Integer.parseInt(parts[2]); - int checksum = Integer.parseInt(parts[3]); - PageToken token = new PageToken(offset, pageSize); - - if (token.hashCode() != checksum) { - throw new IllegalArgumentException("Invalid checksum for token: " + tokenString); - } else { - return token; - } - } catch (Exception e) { - throw new IllegalArgumentException("Failed to decode page token: " + tokenString, e); - } - } - } - - /** - * Builds a new page token to reflect new data that's been read. If the amount of data read is - * less than the pageSize, this will return `PageToken.DONE` (done) - */ - public PageToken updated(List newData) { - if (newData == null || newData.isEmpty() || newData.size() < pageSize) { - return PageToken.DONE; - } else { - return new PageToken(offset + newData.size(), pageSize); - } - } - - /** - * Builds a `PolarisPage` from a `List`. The `PageToken` attached to the new - * `PolarisPage` is the same as the result of calling `updated(data)` on this `PageToken`. - */ - public PolarisPage buildNextPage(List data) { - return new PolarisPage(this.updated(data), data); - } - - /** - * Return a new PageToken with an updated pageSize. If the pageSize provided is null, the existing - * pageSize will be preserved. - */ - public PageToken withPageSize(Integer pageSize) { - if (pageSize == null) { - return new PageToken(this.offset, this.pageSize); - } else { - return new PageToken(this.offset, pageSize); - } - } - - /** Serialize a PageToken into a string */ - @Override - public String toString() { - String tokenContent = TOKEN_PREFIX + ":" + offset + ":" + pageSize + ":" + hashCode(); - return Base64.getEncoder().encodeToString(tokenContent.getBytes(StandardCharsets.UTF_8)); - } - - @Override - public boolean equals(Object o) { - if (o instanceof PageToken) { - PageToken other = (PageToken) o; - return offset == other.offset && pageSize == other.pageSize; - } else { - return false; - } - } - - @Override - public int hashCode() { - return offset + pageSize; - } -} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java new file mode 100644 index 000000000..6a355c572 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.catalog.pagination; + +import java.util.List; + +// TODO implement and comment +public class EntityIdPageToken extends PageToken { + public int id; + + @Override + public PageTokenBuilder builder() { + return null; + } + + @Override + protected List getComponents() { + return List.of(); + } + + public static class EntityIdPageTokenBuilder extends PageTokenBuilder { + + @Override + public String tokenPrefix() { + return ""; + } + + @Override + public int expectedComponents() { + return 0; + } + + @Override + public EntityIdPageToken readEverything() { + return null; + } + + @Override + protected EntityIdPageToken fromStringComponents(List components) { + return null; + } + + @Override + public EntityIdPageToken fromLimit(int limit) { + return null; + } + } + + @Override + public PageToken updated(List newData) { + return null; + } + + @Override + public PageToken withPageSize(Integer pageSize) { + return null; + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java new file mode 100644 index 000000000..737676a7e --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.catalog.pagination; + +import com.sun.jersey.core.util.Base64; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +/** A PageToken implementation that uses an offset to manage pagination. */ +public class OffsetPageToken extends PageToken { + + /** + * The offset of the token. If this is `5` for example, the first 5 entities returned by a list + * operation that uses this token will be skipped. + */ + public final int offset; + + /** The offset to use to start with. */ + private static final int BASE_OFFSET = 0; + + public OffsetPageToken(int offset, int pageSize) { + this.offset = offset; + this.pageSize = pageSize; + } + + @Override + public PageTokenBuilder builder() { + return new OffsetPageTokenBuilder(); + } + + @Override + protected List getComponents() { + return List.of(String.valueOf(this.offset), String.valueOf(this.pageSize)); + } + + public static class OffsetPageTokenBuilder extends PageTokenBuilder { + + @Override + public String tokenPrefix() { + return "polaris-offset"; + } + + @Override + public int expectedComponents() { + // offset + limit + return 2; + } + + @Override + public OffsetPageToken readEverything() { + return new OffsetPageToken(BASE_OFFSET, Integer.MAX_VALUE); + } + + @Override + protected OffsetPageToken fromStringComponents(List components) { + OffsetPageToken token = + new OffsetPageToken( + Integer.parseInt(components.get(1)), Integer.parseInt(components.get(2))); + + if (token.hashCode() != Integer.parseInt(components.get(3))) { + throw new IllegalArgumentException( + "Invalid checksum for offset token: " + token.toString()); + } + return token; + } + + @Override + public OffsetPageToken fromLimit(int limit) { + return new OffsetPageToken(BASE_OFFSET, limit); + } + } + + @Override + public OffsetPageToken updated(List newData) { + return new OffsetPageToken(this.offset + newData.size(), pageSize); + } + + @Override + public OffsetPageToken withPageSize(Integer pageSize) { + return new OffsetPageToken(this.offset, pageSize); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java new file mode 100644 index 000000000..f4f5a1322 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.catalog.pagination; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Represents a page token that can be used by operations like `listTables`. Clients that specify a + * `pageSize` (or a `pageToken`) may receive a `next-page-token` in the response, the content of + * which is a serialized PageToken. + * + *

By providing that in the next query's `pageToken`, the client can resume listing where they + * left off. If the client provides a `pageToken` or `pageSize` but `next-page-token` is null in the + * response, that means there is no more data to read. + */ +public abstract class PageToken { + + public int pageSize; + + private static final int DEFAULT_PAGE_SIZE = 1000; + + public static final PageToken DONE = null; + + /** Get a `PageTokenBuilder` implementation for this `PageToken` implementation */ + public abstract PageTokenBuilder builder(); + + /** Allows `PageToken` implementations to implement methods like `fromLimit` */ + public abstract static class PageTokenBuilder { + + /** + * A prefix that tokens are expected to start with, ideally unique across `PageTokenBuilder` + * implementations. + */ + public abstract String tokenPrefix(); + + /** + * The number of expected components in a token. This should match the number of + * components returned by getComponents and shouldn't account for the prefix + * or the checksum. + */ + public abstract int expectedComponents(); + + /** Construct a `PageToken` to read everything */ + public abstract T readEverything(); + + /** Deserialize a string into a `PageToken` */ + public final T fromString(String tokenString) { + if (tokenString == null) { + return readEverything(); + } else if (tokenString.isEmpty()) { + return fromLimit(DEFAULT_PAGE_SIZE); + } else { + try { + String decoded = + new String(Base64.getDecoder().decode(tokenString), StandardCharsets.UTF_8); + String[] parts = decoded.split(":"); + + // +2 to account for the prefix and checksum. + if (parts.length != expectedComponents() + 2|| !parts[0].equals(tokenPrefix())) { + throw new IllegalArgumentException("Invalid token format in token: " + tokenString); + } + + return fromStringComponents(Arrays.asList(parts)); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to decode page token: " + tokenString, e); + } + } + } + + /** + * PageTokenBuilder implementations should implement this to build a PageToken from components + * in a string token. These components should be the same ones returned by `getComponents` and + * won't include the token prefix or the checksum. + */ + protected abstract T fromStringComponents(List components); + + /** Construct a `PageToken` from a plain limit */ + public abstract T fromLimit(int limit); + } + + /** + * Convert this PageToken to components that the serialized token string will be built from. + */ + protected abstract List getComponents(); + + /** + * Builds a new page token to reflect new data that's been read. If the amount of data read is + * less than the pageSize, this will return `PageToken.DONE` (done) + */ + public abstract PageToken updated(List newData); + + /** + * Builds a `PolarisPage` from a `List`. The `PageToken` attached to the new + * `PolarisPage` is the same as the result of calling `updated(data)` on this `PageToken`. + */ + public final PolarisPage buildNextPage(List data) { + return new PolarisPage(updated(data), data); + } + + /** + * Return a new PageToken with an updated pageSize. If the pageSize provided is null, the existing + * pageSize will be preserved. + */ + public abstract PageToken withPageSize(Integer pageSize); + + /** Serialize a PageToken into a string */ + @Override + public final String toString() { + List components = getComponents(); + String prefix = builder().tokenPrefix(); + String componentString = String.join(":", components); + String checksum = String.valueOf(componentString.hashCode()); + String rawString = prefix + componentString + checksum; + return Base64.getEncoder().encodeToString(rawString.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public final boolean equals(Object o) { + return this.toString().equals(o.toString()); + } + + @Override + public final int hashCode() { + return toString().hashCode(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PolarisPage.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java similarity index 96% rename from polaris-core/src/main/java/org/apache/polaris/core/catalog/PolarisPage.java rename to polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java index c8382e024..06e9ef054 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/PolarisPage.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.core.catalog; +package org.apache.polaris.core.catalog.pagination; import java.util.List; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java index 3041be341..48cb8016b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java @@ -28,8 +28,8 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.catalog.PageToken; -import org.apache.polaris.core.catalog.PolarisPage; +import org.apache.polaris.core.catalog.pagination.PageToken; +import org.apache.polaris.core.catalog.pagination.PolarisPage; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntity; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 1bb865542..2a1ec8efd 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -35,8 +35,9 @@ import java.util.function.Function; import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.catalog.PageToken; -import org.apache.polaris.core.catalog.PolarisPage; +import org.apache.polaris.core.catalog.pagination.OffsetPageToken; +import org.apache.polaris.core.catalog.pagination.PageToken; +import org.apache.polaris.core.catalog.pagination.PolarisPage; import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; @@ -1490,7 +1491,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str catalogId, catalogId, PolarisEntityType.CATALOG_ROLE, - PageToken.fromLimit(2), + ms.pageTokenBuilder().fromLimit(2), entity -> true, Function.identity()) .data; @@ -2018,7 +2019,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( callCtx, () -> this.loadEntity(callCtx, ms, entityCatalogId, entityId)); } - /** Refer to {@link #loadTasks(PolarisCallContext, String, int)} */ + /** Refer to {@link #loadTasks(PolarisCallContext, String, int, PageToken)} */ private @NotNull EntitiesResult loadTasks( @NotNull PolarisCallContext callCtx, @NotNull PolarisMetaStoreSession ms, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index ee74f0730..b36c6a9e1 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -23,8 +23,8 @@ import java.util.function.Predicate; import java.util.function.Supplier; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.catalog.PageToken; -import org.apache.polaris.core.catalog.PolarisPage; +import org.apache.polaris.core.catalog.pagination.PageToken; +import org.apache.polaris.core.catalog.pagination.PolarisPage; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; @@ -528,4 +528,13 @@ boolean hasChildren( /** Rollback the current transaction */ void rollback(); + + /** + * This method is used to construct page tokens when the metastore may need them. Different metastore + * implementations may bring their own PageToken implementations or share them. + * @return A `PageToken.PageTokenBuilder` implementation compatible with this + * `PolarisMetaStoreSession` implementation + */ + @NotNull + PageToken.PageTokenBuilder pageTokenBuilder(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java index a6d77e50b..e29d0d331 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java @@ -25,8 +25,9 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.catalog.PageToken; -import org.apache.polaris.core.catalog.PolarisPage; +import org.apache.polaris.core.catalog.pagination.OffsetPageToken; +import org.apache.polaris.core.catalog.pagination.PageToken; +import org.apache.polaris.core.catalog.pagination.PolarisPage; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; @@ -363,9 +364,13 @@ public List lookupEntityActiveBatch( long catalogId, long parentId, @NotNull PolarisEntityType entityType, - PageToken pageToken, + @NotNull PageToken pageToken, @NotNull Predicate entityFilter, @NotNull Function transformer) { + if (!(pageToken instanceof OffsetPageToken)) { + throw new IllegalArgumentException("Unexpected pageToken:" + pageToken); + } + // full range scan under the parent for that type List entities = this.store @@ -374,12 +379,11 @@ public List lookupEntityActiveBatch( this.store.buildPrefixKeyComposite(catalogId, parentId, entityType.getCode())) .stream() .filter(entityFilter) - .skip(pageToken.offset) + .skip(((OffsetPageToken) pageToken).offset) .limit(pageToken.pageSize) .map(transformer) .collect(Collectors.toList()); - return new PolarisPage( - new PageToken(pageToken.offset + entities.size(), pageToken.pageSize), entities); + return pageToken.buildNextPage(entities); } /** {@inheritDoc} */ @@ -580,4 +584,9 @@ PolarisStorageIntegration loadPolarisStorageIntegration( public void rollback() { this.store.rollback(); } + + @Override + public @NotNull PageToken.PageTokenBuilder pageTokenBuilder() { + return new OffsetPageToken.OffsetPageTokenBuilder(); + } } diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java index 53d2b63df..7b3fc9e56 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java @@ -35,7 +35,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.catalog.PageToken; +import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.PolarisBaseEntity; diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java index 7dffe6302..8692ee38c 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java @@ -28,7 +28,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.catalog.PageToken; +import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntity; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index a8410e6ee..01cd70734 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -56,7 +56,7 @@ import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizableOperation; import org.apache.polaris.core.auth.PolarisAuthorizer; -import org.apache.polaris.core.catalog.PageToken; +import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; @@ -723,7 +723,7 @@ private List listCatalogsUnsafe() { null, PolarisEntityType.CATALOG, PolarisEntitySubType.ANY_SUBTYPE, - PageToken.readEverything()) + entityManager.newMetaStoreSession().pageTokenBuilder().readEverything()) .getEntities() .stream() .map( @@ -902,7 +902,7 @@ public List listPrincipals() { null, PolarisEntityType.PRINCIPAL, PolarisEntitySubType.NULL_SUBTYPE, - PageToken.readEverything()) + entityManager.newMetaStoreSession().pageTokenBuilder().readEverything()) .getEntities() .stream() .map( @@ -1025,7 +1025,7 @@ public List listPrincipalRoles() { null, PolarisEntityType.PRINCIPAL_ROLE, PolarisEntitySubType.NULL_SUBTYPE, - PageToken.readEverything()) + entityManager.newMetaStoreSession().pageTokenBuilder().readEverything()) .getEntities() .stream() .map( @@ -1167,7 +1167,7 @@ public List listCatalogRoles(String catalogName) { PolarisEntity.toCoreList(List.of(catalogEntity)), PolarisEntityType.CATALOG_ROLE, PolarisEntitySubType.NULL_SUBTYPE, - PageToken.readEverything()) + entityManager.newMetaStoreSession().pageTokenBuilder().readEverything()) .getEntities() .stream() .map( diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 196a6e60e..d19d0b123 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -78,9 +78,9 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.catalog.PageToken; +import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; -import org.apache.polaris.core.catalog.PolarisPage; +import org.apache.polaris.core.catalog.pagination.PolarisPage; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.NamespaceEntity; @@ -176,6 +176,7 @@ public class BasePolarisCatalog extends BaseMetastoreViewCatalog private Map catalogProperties; private Map tableDefaultProperties; private final FileIOFactory fileIOFactory; + private final PageToken.PageTokenBuilder pageTokenBuilder; /** * @param entityManager provides handle to underlying PolarisMetaStoreManager with which to @@ -202,6 +203,7 @@ public BasePolarisCatalog( this.catalogId = catalogEntity.getId(); this.catalogName = catalogEntity.getName(); this.fileIOFactory = fileIOFactory; + this.pageTokenBuilder = entityManager.newMetaStoreSession().pageTokenBuilder(); } @Override @@ -459,7 +461,7 @@ private boolean paginationEnabled() { @Override public List listTables(Namespace namespace) { - return listTables(namespace, PageToken.readEverything()).data; + return listTables(namespace, pageTokenBuilder.readEverything()).data; } public PolarisPage listTables(Namespace namespace, PageToken pageToken) { @@ -760,7 +762,7 @@ public List listNamespaces() { @Override public List listNamespaces(Namespace namespace) throws NoSuchNamespaceException { - return listNamespaces(namespace, PageToken.readEverything()).data; + return listNamespaces(namespace, pageTokenBuilder.readEverything()).data; } public PolarisPage listNamespaces(PageToken pageToken) @@ -803,7 +805,7 @@ public void close() throws IOException {} @Override public List listViews(Namespace namespace) { - return listViews(namespace, PageToken.readEverything()).data; + return listViews(namespace, pageTokenBuilder.readEverything()).data; } public PolarisPage listViews(Namespace namespace, PageToken pageToken) { @@ -1090,7 +1092,7 @@ private void validateNoLocationOverlap( parentPath.stream().map(PolarisEntity::toCore).collect(Collectors.toList()), PolarisEntityType.NAMESPACE, PolarisEntitySubType.ANY_SUBTYPE, - PageToken.readEverything()); + pageTokenBuilder.readEverything()); if (!siblingNamespacesResult.isSuccess()) { throw new IllegalStateException( "Unable to resolve siblings entities to validate location - could not list namespaces"); @@ -1117,7 +1119,7 @@ private void validateNoLocationOverlap( .collect(Collectors.toList()), PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE, - PageToken.readEverything()); + pageTokenBuilder.readEverything()); if (!siblingTablesResult.isSuccess()) { throw new IllegalStateException( "Unable to resolve siblings entities to validate location - could not list tables"); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 76e58958f..b749eb886 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -49,7 +49,7 @@ import org.apache.iceberg.rest.responses.ConfigResponse; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizer; -import org.apache.polaris.core.catalog.PageToken; +import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisEntity; @@ -127,7 +127,11 @@ public Response listNamespaces( Optional namespaceOptional = Optional.ofNullable(parent).map(IcebergCatalogAdapter::decodeNamespace); - PageToken token = PageToken.fromString(pageToken).withPageSize(pageSize); + PolarisEntityManager entityManager = + entityManagerFactory.getOrCreateEntityManager( + CallContext.getCurrentContext().getRealmContext()); + PageToken token = entityManager.newMetaStoreSession() + .pageTokenBuilder().fromString(pageToken).withPageSize(pageSize); var response = newHandlerWrapper(securityContext, prefix) .listNamespaces(namespaceOptional.orElse(Namespace.of()), token); @@ -216,7 +220,11 @@ public Response listTables( SecurityContext securityContext) { Namespace ns = decodeNamespace(namespace); - PageToken token = PageToken.fromString(pageToken).withPageSize(pageSize); + PolarisEntityManager entityManager = + entityManagerFactory.getOrCreateEntityManager( + CallContext.getCurrentContext().getRealmContext()); + PageToken token = entityManager.newMetaStoreSession() + .pageTokenBuilder().fromString(pageToken).withPageSize(pageSize); return Response.ok(newHandlerWrapper(securityContext, prefix).listTables(ns, token)).build(); } @@ -352,7 +360,11 @@ public Response listViews( Integer pageSize, SecurityContext securityContext) { Namespace ns = decodeNamespace(namespace); - PageToken token = PageToken.fromString(pageToken).withPageSize(pageSize); + PolarisEntityManager entityManager = + entityManagerFactory.getOrCreateEntityManager( + CallContext.getCurrentContext().getRealmContext()); + PageToken token = entityManager.newMetaStoreSession() + .pageTokenBuilder().fromString(pageToken).withPageSize(pageSize); return Response.ok(newHandlerWrapper(securityContext, prefix).listViews(ns, token)).build(); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java index 002457257..4fbba5b71 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java @@ -71,9 +71,9 @@ import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizableOperation; import org.apache.polaris.core.auth.PolarisAuthorizer; -import org.apache.polaris.core.catalog.PageToken; +import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; -import org.apache.polaris.core.catalog.PolarisPage; +import org.apache.polaris.core.catalog.pagination.PolarisPage; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java b/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java index a35323ae3..27986210f 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java @@ -24,8 +24,8 @@ import java.util.List; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.rest.responses.ListNamespacesResponse; -import org.apache.polaris.core.catalog.PageToken; -import org.apache.polaris.core.catalog.PolarisPage; +import org.apache.polaris.core.catalog.pagination.PageToken; +import org.apache.polaris.core.catalog.pagination.PolarisPage; /** * Used in lieu of `ListNamespacesResponse` when there may be a `PageToken` associated with the diff --git a/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java b/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java index 662936f45..7b6dcc3f0 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java @@ -24,8 +24,8 @@ import java.util.List; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.rest.responses.ListTablesResponse; -import org.apache.polaris.core.catalog.PageToken; -import org.apache.polaris.core.catalog.PolarisPage; +import org.apache.polaris.core.catalog.pagination.PageToken; +import org.apache.polaris.core.catalog.pagination.PolarisPage; public class ListTablesResponseWithPageToken extends ListTablesResponse { private final PageToken pageToken; diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index 2cc8a0e40..86d9b2a42 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -62,8 +62,8 @@ import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizer; -import org.apache.polaris.core.catalog.PageToken; -import org.apache.polaris.core.catalog.PolarisPage; +import org.apache.polaris.core.catalog.pagination.PageToken; +import org.apache.polaris.core.catalog.pagination.PolarisPage; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.CatalogEntity; @@ -1124,7 +1124,10 @@ public void testDropTableWithPurge() { .rejects(TABLE); List tasks = metaStoreManager - .loadTasks(polarisContext, "testExecutor", PageToken.fromLimit(1)) + .loadTasks( + polarisContext, + "testExecutor", + polarisContext.getMetaStore().pageTokenBuilder().fromLimit(1)) .getEntities(); Assertions.assertThat(tasks).hasSize(1); TaskEntity taskEntity = TaskEntity.of(tasks.get(0)); @@ -1239,7 +1242,10 @@ public void testFileIOWrapper() { handler.handleTask( TaskEntity.of( metaStoreManager - .loadTasks(polarisContext, "testExecutor", PageToken.fromLimit(1)) + .loadTasks( + polarisContext, + "testExecutor", + polarisContext.getMetaStore().pageTokenBuilder().fromLimit(1)) .getEntities() .getFirst())); Assertions.assertThat(measured.getNumDeletedFiles()).as("A table was deleted").isGreaterThan(0); @@ -1260,7 +1266,9 @@ public void testPaginatedListTables() { Assertions.assertThat(catalog.listTables(NS)).isNotNull().hasSize(5); // List with a limit: - PolarisPage firstListResult = catalog.listTables(NS, PageToken.fromLimit(2)); + PolarisPage firstListResult = catalog.listTables( + NS, + polarisContext.getMetaStore().pageTokenBuilder().fromLimit(2)); Assertions.assertThat(firstListResult.data.size()).isEqualTo(2); Assertions.assertThat(firstListResult.pageToken.toString()).isNotNull().isNotEmpty(); @@ -1295,7 +1303,9 @@ public void testPaginatedListViews() { Assertions.assertThat(catalog.listViews(NS)).isNotNull().hasSize(5); // List with a limit: - PolarisPage firstListResult = catalog.listViews(NS, PageToken.fromLimit(2)); + PolarisPage firstListResult = catalog.listViews( + NS, + polarisContext.getMetaStore().pageTokenBuilder().fromLimit(2)); Assertions.assertThat(firstListResult.data.size()).isEqualTo(2); Assertions.assertThat(firstListResult.pageToken.toString()).isNotNull().isNotEmpty(); @@ -1326,7 +1336,8 @@ public void testPaginatedListNamespaces() { Assertions.assertThat(catalog.listNamespaces()).isNotNull().hasSize(5); // List with a limit: - PolarisPage firstListResult = catalog.listNamespaces(PageToken.fromLimit(2)); + PolarisPage firstListResult = catalog.listNamespaces( + polarisContext.getMetaStore().pageTokenBuilder().fromLimit(2)); Assertions.assertThat(firstListResult.data.size()).isEqualTo(2); Assertions.assertThat(firstListResult.pageToken.toString()).isNotNull().isNotEmpty(); @@ -1341,7 +1352,8 @@ public void testPaginatedListNamespaces() { Assertions.assertThat(finalListResult.pageToken).isNull(); // List with page size matching the amount of data - PolarisPage firstExactListResult = catalog.listNamespaces(PageToken.fromLimit(5)); + PolarisPage firstExactListResult = catalog.listNamespaces( + polarisContext.getMetaStore().pageTokenBuilder().fromLimit(5)); Assertions.assertThat(firstExactListResult.data.size()).isEqualTo(5); Assertions.assertThat(firstExactListResult.pageToken.toString()).isNotNull().isNotEmpty(); @@ -1351,7 +1363,8 @@ public void testPaginatedListNamespaces() { Assertions.assertThat(secondExactListResult.pageToken).isNull(); // List with huge page size: - PolarisPage bigListResult = catalog.listNamespaces(PageToken.fromLimit(9999)); + PolarisPage bigListResult = catalog.listNamespaces( + polarisContext.getMetaStore().pageTokenBuilder().fromLimit(9999)); Assertions.assertThat(bigListResult.data.size()).isEqualTo(5); Assertions.assertThat(bigListResult.pageToken).isNull(); } finally { diff --git a/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java b/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java index 75e48dec8..448e91353 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java @@ -32,7 +32,8 @@ import org.apache.iceberg.io.FileIO; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; -import org.apache.polaris.core.catalog.PageToken; +import org.apache.polaris.core.catalog.pagination.OffsetPageToken; +import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.AsyncTaskType; @@ -92,7 +93,10 @@ public void testTableCleanup() throws IOException { assertThat( metaStoreManagerFactory .getOrCreateMetaStoreManager(realmContext) - .loadTasks(polarisCallContext, "test", PageToken.fromLimit(1)) + .loadTasks( + polarisCallContext, + "test", + new OffsetPageToken.OffsetPageTokenBuilder().fromLimit(1)) .getEntities()) .hasSize(1) .satisfiesExactly( @@ -167,7 +171,10 @@ public void close() { assertThat( metaStoreManagerFactory .getOrCreateMetaStoreManager(realmContext) - .loadTasks(polarisCallContext, "test", PageToken.fromLimit(5)) + .loadTasks( + polarisCallContext, + "test", + new OffsetPageToken.OffsetPageTokenBuilder().fromLimit(5)) .getEntities()) .hasSize(1); } @@ -237,7 +244,10 @@ public void close() { assertThat( metaStoreManagerFactory .getOrCreateMetaStoreManager(realmContext) - .loadTasks(polarisCallContext, "test", PageToken.fromLimit(5)) + .loadTasks( + polarisCallContext, + "test", + new OffsetPageToken.OffsetPageTokenBuilder().fromLimit(5)) .getEntities()) .hasSize(2) .satisfiesExactly( @@ -327,7 +337,10 @@ public void testTableCleanupMultipleSnapshots() throws IOException { assertThat( metaStoreManagerFactory .getOrCreateMetaStoreManager(realmContext) - .loadTasks(polarisCallContext, "test", PageToken.fromLimit(5)) + .loadTasks( + polarisCallContext, + "test", + new OffsetPageToken.OffsetPageTokenBuilder().fromLimit(5)) .getEntities()) // all three manifests should be present, even though one is excluded from the latest // snapshot From 1100f71ea2d747d67ccdbf961536b5d904b31662 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 9 Sep 2024 21:34:27 -0400 Subject: [PATCH 20/32] check in after major refactor --- ...olarisEclipseLinkMetaStoreSessionImpl.java | 3 +- .../eclipselink/PolarisEclipseLinkStore.java | 27 ++- .../catalog/pagination/EntityIdPageToken.java | 62 ++++-- .../catalog/pagination/OffsetPageToken.java | 53 +++-- .../core/catalog/pagination/PageToken.java | 69 +++--- .../pagination/ReadEverythingPageToken.java | 87 ++++++++ .../PolarisMetaStoreManagerImpl.java | 1 - .../persistence/PolarisMetaStoreSession.java | 7 +- .../PolarisTreeMapMetaStoreSessionImpl.java | 28 ++- .../catalog/pagination/PageTokenTest.java | 200 ++++++++++++++++++ .../BasePolarisMetaStoreManagerTest.java | 32 ++- .../PolarisTestMetaStoreManager.java | 14 +- .../service/admin/PolarisAdminService.java | 10 +- .../service/catalog/BasePolarisCatalog.java | 13 +- .../catalog/IcebergCatalogAdapter.java | 24 ++- .../catalog/PolarisCatalogHandlerWrapper.java | 2 +- .../catalog/BasePolarisCatalogTest.java | 30 +-- .../task/TableCleanupTaskHandlerTest.java | 21 +- 18 files changed, 535 insertions(+), 148 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java create mode 100644 polaris-core/src/test/java/org/apache/polaris/core/catalog/pagination/PageTokenTest.java diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index bf6c4dd57..5c0d871de 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -553,6 +553,7 @@ public List lookupEntityActiveBatch( this.store .lookupFullEntitiesActive( localSession.get(), catalogId, parentId, entityType, pageToken) + .stream() .map(ModelEntity::toEntity) .filter(entityFilter) .map(transformer) @@ -770,6 +771,6 @@ public void rollback() { @Override public @NotNull PageToken.PageTokenBuilder pageTokenBuilder() { - return new EntityIdPageToken.EntityIdPageTokenBuilder(); + return EntityIdPageToken.builder(); } } diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java index ede128412..6d79fb506 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java @@ -23,10 +23,10 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.catalog.pagination.EntityIdPageToken; import org.apache.polaris.core.catalog.pagination.PageToken; +import org.apache.polaris.core.catalog.pagination.ReadEverythingPageToken; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; import org.apache.polaris.core.entity.PolarisEntityActiveRecord; @@ -276,18 +276,25 @@ long countActiveChildEntities( return query.getSingleResult(); } - Stream lookupFullEntitiesActive( + List lookupFullEntitiesActive( EntityManager session, long catalogId, long parentId, @NotNull PolarisEntityType entityType, @NotNull PageToken pageToken) { diagnosticServices.check(session != null, "session_is_null"); - diagnosticServices.check(pageToken instanceof EntityIdPageToken, "unexpected_page_token"); + diagnosticServices.check( + (pageToken instanceof EntityIdPageToken || pageToken instanceof ReadEverythingPageToken), + "unexpected_page_token"); // Currently check against ENTITIES not joining with ENTITIES_ACTIVE String hql = - "SELECT m from ModelEntity m where m.catalogId=:catalogId and m.parentId=:parentId and m.typeCode=:typeCode and m.id > :tokenId"; + "SELECT m from ModelEntity m " + + " where m.catalogId=:catalogId and m.parentId=:parentId and m.typeCode=:typeCode and m.id > :tokenId"; + + if (pageToken instanceof EntityIdPageToken) { + hql += " ORDER BY m.id ASC"; + } TypedQuery query = session @@ -295,10 +302,16 @@ Stream lookupFullEntitiesActive( .setParameter("catalogId", catalogId) .setParameter("parentId", parentId) .setParameter("typeCode", entityType.getCode()) - .setParameter("tokenId", ((EntityIdPageToken) pageToken).id) - .setMaxResults(pageToken.pageSize); + .setParameter("tokenId", -1L); + + if (pageToken instanceof EntityIdPageToken) { + query = + query + .setParameter("tokenId", ((EntityIdPageToken) pageToken).id) + .setMaxResults(pageToken.pageSize); + } - return query.getResultStream(); + return query.getResultList(); } ModelEntityDropped lookupEntityDropped(EntityManager session, long catalogId, long entityId) { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java index 6a355c572..18a3c6fca 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java @@ -19,56 +19,86 @@ package org.apache.polaris.core.catalog.pagination; import java.util.List; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.persistence.models.ModelEntity; // TODO implement and comment public class EntityIdPageToken extends PageToken { - public int id; + public long id; - @Override - public PageTokenBuilder builder() { - return null; + private EntityIdPageToken(long id, int pageSize) { + this.id = id; + this.pageSize = pageSize; + validate(); } + /** The entity ID to use to start with. */ + private static final long BASE_ID = -1L; + @Override protected List getComponents() { - return List.of(); + return List.of(String.valueOf(id), String.valueOf(pageSize)); + } + + /** Get a new `EntityIdPageTokenBuilder` instance */ + public static PageTokenBuilder builder() { + return new EntityIdPageToken.EntityIdPageTokenBuilder(); + } + + @Override + protected PageTokenBuilder getBuilder() { + return EntityIdPageToken.builder(); } public static class EntityIdPageTokenBuilder extends PageTokenBuilder { @Override public String tokenPrefix() { - return ""; + return "polaris-entity-id"; } @Override public int expectedComponents() { - return 0; - } - - @Override - public EntityIdPageToken readEverything() { - return null; + // id, pageSize + return 2; } @Override protected EntityIdPageToken fromStringComponents(List components) { - return null; + return new EntityIdPageToken( + Integer.parseInt(components.get(0)), Integer.parseInt(components.get(1))); } @Override public EntityIdPageToken fromLimit(int limit) { - return null; + return new EntityIdPageToken(BASE_ID, limit); } } @Override public PageToken updated(List newData) { - return null; + if (newData == null || newData.size() < this.pageSize) { + return PageToken.DONE; + } else { + if (newData.get(0) instanceof ModelEntity) { + return new EntityIdPageToken( + ((ModelEntity) newData.get(newData.size() - 1)).getId(), this.pageSize); + } else if (newData.get(0) instanceof PolarisBaseEntity) { + return new EntityIdPageToken( + ((PolarisBaseEntity) newData.get(newData.size() - 1)).getId(), this.pageSize); + } else { + throw new IllegalArgumentException( + "Cannot build a page token from: " + newData.get(0).getClass().getSimpleName()); + } + } } @Override public PageToken withPageSize(Integer pageSize) { - return null; + if (pageSize == null) { + return new EntityIdPageToken(BASE_ID, this.pageSize); + } else { + return new EntityIdPageToken(this.id, pageSize); + } } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java index 737676a7e..0ef83f7e9 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java @@ -18,9 +18,6 @@ */ package org.apache.polaris.core.catalog.pagination; -import com.sun.jersey.core.util.Base64; - -import java.nio.charset.StandardCharsets; import java.util.List; /** A PageToken implementation that uses an offset to manage pagination. */ @@ -35,16 +32,30 @@ public class OffsetPageToken extends PageToken { /** The offset to use to start with. */ private static final int BASE_OFFSET = 0; - public OffsetPageToken(int offset, int pageSize) { + private OffsetPageToken(int offset, int pageSize) { this.offset = offset; this.pageSize = pageSize; + validate(); } @Override - public PageTokenBuilder builder() { + protected void validate() { + if (offset < 0) { + throw new IllegalArgumentException("Offset must be greater than zero"); + } + super.validate(); + } + + /** Get a new `EntityIdPageTokenBuilder` instance */ + public static PageTokenBuilder builder() { return new OffsetPageTokenBuilder(); } + @Override + protected PageTokenBuilder getBuilder() { + return OffsetPageToken.builder(); + } + @Override protected List getComponents() { return List.of(String.valueOf(this.offset), String.valueOf(this.pageSize)); @@ -52,6 +63,8 @@ protected List getComponents() { public static class OffsetPageTokenBuilder extends PageTokenBuilder { + private OffsetPageTokenBuilder() {} + @Override public String tokenPrefix() { return "polaris-offset"; @@ -63,22 +76,10 @@ public int expectedComponents() { return 2; } - @Override - public OffsetPageToken readEverything() { - return new OffsetPageToken(BASE_OFFSET, Integer.MAX_VALUE); - } - @Override protected OffsetPageToken fromStringComponents(List components) { - OffsetPageToken token = - new OffsetPageToken( - Integer.parseInt(components.get(1)), Integer.parseInt(components.get(2))); - - if (token.hashCode() != Integer.parseInt(components.get(3))) { - throw new IllegalArgumentException( - "Invalid checksum for offset token: " + token.toString()); - } - return token; + return new OffsetPageToken( + Integer.parseInt(components.get(0)), Integer.parseInt(components.get(1))); } @Override @@ -88,12 +89,20 @@ public OffsetPageToken fromLimit(int limit) { } @Override - public OffsetPageToken updated(List newData) { - return new OffsetPageToken(this.offset + newData.size(), pageSize); + public PageToken updated(List newData) { + if (newData == null || newData.size() < this.pageSize) { + return PageToken.DONE; + } else { + return new OffsetPageToken(this.offset + newData.size(), pageSize); + } } @Override public OffsetPageToken withPageSize(Integer pageSize) { - return new OffsetPageToken(this.offset, pageSize); + if (pageSize == null) { + return new OffsetPageToken(BASE_OFFSET, this.pageSize); + } else { + return new OffsetPageToken(this.offset, pageSize); + } } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java index f4f5a1322..b38eafe67 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java @@ -19,11 +19,11 @@ package org.apache.polaris.core.catalog.pagination; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Represents a page token that can be used by operations like `listTables`. Clients that specify a @@ -38,12 +38,21 @@ public abstract class PageToken { public int pageSize; - private static final int DEFAULT_PAGE_SIZE = 1000; - public static final PageToken DONE = null; + public static final int DEFAULT_PAGE_SIZE = 1000; + + protected void validate() { + if (pageSize <= 0) { + throw new IllegalArgumentException("Page size must be greater than zero"); + } + } - /** Get a `PageTokenBuilder` implementation for this `PageToken` implementation */ - public abstract PageTokenBuilder builder(); + /** + * Get a new PageTokenBuilder from a PageToken. The PageTokenBuilder type should match the + * PageToken type. Implementations may also provide a static `builder` method to obtain the same + * PageTokenBuilder. + */ + protected abstract PageTokenBuilder getBuilder(); /** Allows `PageToken` implementations to implement methods like `fromLimit` */ public abstract static class PageTokenBuilder { @@ -55,21 +64,21 @@ public abstract static class PageTokenBuilder { public abstract String tokenPrefix(); /** - * The number of expected components in a token. This should match the number of - * components returned by getComponents and shouldn't account for the prefix - * or the checksum. + * The number of expected components in a token. This should match the number of components + * returned by getComponents and shouldn't account for the prefix or the checksum. */ public abstract int expectedComponents(); - /** Construct a `PageToken` to read everything */ - public abstract T readEverything(); - /** Deserialize a string into a `PageToken` */ - public final T fromString(String tokenString) { + public final PageToken fromString(String tokenString) { if (tokenString == null) { - return readEverything(); + return ReadEverythingPageToken.get(); } else if (tokenString.isEmpty()) { - return fromLimit(DEFAULT_PAGE_SIZE); + if (this instanceof ReadEverythingPageToken.ReadEverythingPageTokenBuilder) { + return ReadEverythingPageToken.get(); + } else { + return fromLimit(DEFAULT_PAGE_SIZE); + } } else { try { String decoded = @@ -77,11 +86,13 @@ public final T fromString(String tokenString) { String[] parts = decoded.split(":"); // +2 to account for the prefix and checksum. - if (parts.length != expectedComponents() + 2|| !parts[0].equals(tokenPrefix())) { + if (parts.length != expectedComponents() + 2 || !parts[0].equals(tokenPrefix())) { throw new IllegalArgumentException("Invalid token format in token: " + tokenString); } - return fromStringComponents(Arrays.asList(parts)); + T result = fromStringComponents(Arrays.asList(parts).subList(1, parts.length - 1)); + result.validate(); + return result; } catch (Exception e) { throw new IllegalArgumentException("Failed to decode page token: " + tokenString, e); } @@ -99,16 +110,14 @@ public final T fromString(String tokenString) { public abstract T fromLimit(int limit); } - /** - * Convert this PageToken to components that the serialized token string will be built from. - */ + /** Convert this PageToken to components that the serialized token string will be built from. */ protected abstract List getComponents(); /** * Builds a new page token to reflect new data that's been read. If the amount of data read is * less than the pageSize, this will return `PageToken.DONE` (done) */ - public abstract PageToken updated(List newData); + protected abstract PageToken updated(List newData); /** * Builds a `PolarisPage` from a `List`. The `PageToken` attached to the new @@ -128,20 +137,32 @@ public final PolarisPage buildNextPage(List data) { @Override public final String toString() { List components = getComponents(); - String prefix = builder().tokenPrefix(); + String prefix = getBuilder().tokenPrefix(); String componentString = String.join(":", components); String checksum = String.valueOf(componentString.hashCode()); - String rawString = prefix + componentString + checksum; + List allElements = + Stream.of(prefix, componentString, checksum) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + String rawString = String.join(":", allElements); return Base64.getEncoder().encodeToString(rawString.getBytes(StandardCharsets.UTF_8)); } @Override public final boolean equals(Object o) { - return this.toString().equals(o.toString()); + if (o instanceof PageToken) { + return this.toString().equals(o.toString()); + } else { + return false; + } } @Override public final int hashCode() { - return toString().hashCode(); + if (toString() == null) { + return 0; + } else { + return toString().hashCode(); + } } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java new file mode 100644 index 000000000..dbfa3b4a7 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.catalog.pagination; + +import java.util.List; + +/** + * A PageToken implementation for readers who want to read everything. The behavior when using this + * token should be the same as when reading without a token. + */ +public class ReadEverythingPageToken extends PageToken { + + private ReadEverythingPageToken() { + this.pageSize = Integer.MAX_VALUE; + validate(); + } + + public static ReadEverythingPageToken get() { + return new ReadEverythingPageToken(); + } + + public static PageTokenBuilder builder() { + return new ReadEverythingPageTokenBuilder(); + } + + @Override + protected PageTokenBuilder getBuilder() { + return builder(); + } + + public static class ReadEverythingPageTokenBuilder + extends PageTokenBuilder { + + private ReadEverythingPageTokenBuilder() {} + + @Override + public String tokenPrefix() { + return "polaris-read-everything"; + } + + @Override + public int expectedComponents() { + return 0; + } + + @Override + protected ReadEverythingPageToken fromStringComponents(List components) { + return ReadEverythingPageToken.get(); + } + + @Override + public ReadEverythingPageToken fromLimit(int limit) { + throw new UnsupportedOperationException(); + } + } + + @Override + protected List getComponents() { + return List.of(); + } + + @Override + public PageToken updated(List newData) { + return PageToken.DONE; + } + + @Override + public PageToken withPageSize(Integer pageSize) { + throw new UnsupportedOperationException(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 2a1ec8efd..04003c7fd 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -35,7 +35,6 @@ import java.util.function.Function; import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.catalog.pagination.OffsetPageToken; import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.catalog.pagination.PolarisPage; import org.apache.polaris.core.entity.AsyncTaskType; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index b36c6a9e1..2fc233308 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -530,10 +530,11 @@ boolean hasChildren( void rollback(); /** - * This method is used to construct page tokens when the metastore may need them. Different metastore - * implementations may bring their own PageToken implementations or share them. + * This method is used to construct page tokens when the metastore may need them. Different + * metastore implementations may bring their own PageToken implementations or share them. + * * @return A `PageToken.PageTokenBuilder` implementation compatible with this - * `PolarisMetaStoreSession` implementation + * `PolarisMetaStoreSession` implementation */ @NotNull PageToken.PageTokenBuilder pageTokenBuilder(); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java index e29d0d331..328a4777a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java @@ -19,15 +19,18 @@ package org.apache.polaris.core.persistence; import com.google.common.base.Predicates; +import java.util.Comparator; import java.util.List; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.catalog.pagination.OffsetPageToken; import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.catalog.pagination.PolarisPage; +import org.apache.polaris.core.catalog.pagination.ReadEverythingPageToken; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; @@ -367,22 +370,29 @@ public List lookupEntityActiveBatch( @NotNull PageToken pageToken, @NotNull Predicate entityFilter, @NotNull Function transformer) { - if (!(pageToken instanceof OffsetPageToken)) { - throw new IllegalArgumentException("Unexpected pageToken:" + pageToken); + if (!(pageToken instanceof ReadEverythingPageToken) + && !(pageToken instanceof OffsetPageToken)) { + throw new IllegalArgumentException("Unexpected pageToken: " + pageToken); } // full range scan under the parent for that type - List entities = + Stream partialResults = this.store .getSliceEntitiesActive() .readRange( this.store.buildPrefixKeyComposite(catalogId, parentId, entityType.getCode())) .stream() - .filter(entityFilter) - .skip(((OffsetPageToken) pageToken).offset) - .limit(pageToken.pageSize) - .map(transformer) - .collect(Collectors.toList()); + .filter(entityFilter); + + if (pageToken instanceof OffsetPageToken) { + partialResults = + partialResults + .sorted(Comparator.comparingLong(PolarisEntityCore::getId)) + .skip(((OffsetPageToken) pageToken).offset) + .limit(pageToken.pageSize); + } + + List entities = partialResults.map(transformer).collect(Collectors.toList()); return pageToken.buildNextPage(entities); } @@ -587,6 +597,6 @@ public void rollback() { @Override public @NotNull PageToken.PageTokenBuilder pageTokenBuilder() { - return new OffsetPageToken.OffsetPageTokenBuilder(); + return OffsetPageToken.builder(); } } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/catalog/pagination/PageTokenTest.java b/polaris-core/src/test/java/org/apache/polaris/core/catalog/pagination/PageTokenTest.java new file mode 100644 index 000000000..d6ed225fa --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/catalog/pagination/PageTokenTest.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.catalog.pagination; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.polaris.core.persistence.models.ModelEntity; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PageTokenTest { + private static final Logger LOGGER = LoggerFactory.getLogger(PageTokenTest.class); + + static Stream> getPageTokenBuilders() { + return Stream.of( + OffsetPageToken.builder(), EntityIdPageToken.builder(), ReadEverythingPageToken.builder()); + } + + @Test + void testDoneToken() { + Assertions.assertThat(PageToken.DONE).isNull(); + } + + @ParameterizedTest + @MethodSource("getPageTokenBuilders") + void testRoundTrips(PageToken.PageTokenBuilder builder) { + if (builder instanceof ReadEverythingPageToken.ReadEverythingPageTokenBuilder) { + // Skip ReadEverythingPageToken + return; + } + + for (int limit : List.of(1, 10, 100, Integer.MAX_VALUE)) { + PageToken token = builder.fromLimit(limit); + Assertions.assertThat(token.pageSize).isEqualTo(limit); + Assertions.assertThat(builder.fromString(token.toString())).isEqualTo(token); + } + } + + @ParameterizedTest + @MethodSource("getPageTokenBuilders") + void testInvalidLimits(PageToken.PageTokenBuilder builder) { + if (builder instanceof ReadEverythingPageToken.ReadEverythingPageTokenBuilder) { + // Skip ReadEverythingPageToken + return; + } + + for (int limit : List.of(-1, 0)) { + Assertions.assertThatThrownBy(() -> builder.fromLimit(limit)) + .isInstanceOf(IllegalArgumentException.class); + } + } + + @ParameterizedTest + @MethodSource("getPageTokenBuilders") + void testStartingTokens(PageToken.PageTokenBuilder builder) { + Assertions.assertThat(builder.fromString(null)).isNotNull(); + Assertions.assertThat(builder.fromString(null)).isEqualTo(ReadEverythingPageToken.get()); + + Assertions.assertThat(builder.fromString("")).isNotNull(); + if (!(builder instanceof ReadEverythingPageToken.ReadEverythingPageTokenBuilder)) { + Assertions.assertThat(builder.fromString("")).isNotEqualTo(ReadEverythingPageToken.get()); + } + } + + @ParameterizedTest + @MethodSource("getPageTokenBuilders") + void testPageBuilding(PageToken.PageTokenBuilder builder) { + if (builder instanceof ReadEverythingPageToken.ReadEverythingPageTokenBuilder) { + // Skip ReadEverythingPageToken + return; + } + + List data = + List.of(ModelEntity.builder().id(1).build(), ModelEntity.builder().id(2).build()); + + PageToken token = builder.fromLimit(1000); + Assertions.assertThat(token.buildNextPage(data).data).isEqualTo(data); + Assertions.assertThat(token.buildNextPage(data).pageToken).isNull(); + } + + @Test + void testUniquePrefixes() { + Stream> builders = getPageTokenBuilders(); + List prefixes = + builders.map(PageToken.PageTokenBuilder::tokenPrefix).collect(Collectors.toList()); + Assertions.assertThat(prefixes.size()).isEqualTo(prefixes.stream().distinct().count()); + } + + @ParameterizedTest + @MethodSource("getPageTokenBuilders") + void testCrossTokenParsing(PageToken.PageTokenBuilder builder) { + var otherBuilders = getPageTokenBuilders().collect(Collectors.toList()); + for (var otherBuilder : otherBuilders) { + LOGGER.info( + "Testing {} being parsed by {}", + builder.getClass().getSimpleName(), + otherBuilder.getClass().getSimpleName()); + + final PageToken token; + if (builder instanceof ReadEverythingPageToken.ReadEverythingPageTokenBuilder) { + token = ReadEverythingPageToken.get(); + } else { + token = builder.fromLimit(1234); + } + if (otherBuilder.getClass().equals(builder.getClass())) { + Assertions.assertThat(otherBuilder.fromString(token.toString())).isEqualTo(token); + } else { + Assertions.assertThatThrownBy(() -> otherBuilder.fromString(token.toString())) + .isInstanceOf(IllegalArgumentException.class); + } + } + } + + @ParameterizedTest + @MethodSource("getPageTokenBuilders") + void testDefaultTokens(PageToken.PageTokenBuilder builder) { + if (builder instanceof ReadEverythingPageToken.ReadEverythingPageTokenBuilder) { + // Skip ReadEverythingPageToken + return; + } + + PageToken token = builder.fromString(""); + Assertions.assertThat(token.toString()).isNotNull(); + Assertions.assertThat(token.pageSize).isEqualTo(PageToken.DEFAULT_PAGE_SIZE); + } + + @Test + void testReadEverythingPageToken() { + PageToken token = ReadEverythingPageToken.get(); + + Assertions.assertThat(token.toString()).isNotNull(); + Assertions.assertThat(token.updated(List.of("anything"))).isEqualTo(PageToken.DONE); + Assertions.assertThat(token.pageSize).isEqualTo(Integer.MAX_VALUE); + } + + @Test + void testOffsetPageToken() { + OffsetPageToken token = OffsetPageToken.builder().fromLimit(2); + + Assertions.assertThat(token).isInstanceOf(OffsetPageToken.class); + Assertions.assertThat(token.offset).isEqualTo(0); + + List data = List.of("some", "data"); + var page = token.buildNextPage(data); + Assertions.assertThat(page.pageToken).isNotNull(); + Assertions.assertThat(page.pageToken).isInstanceOf(OffsetPageToken.class); + Assertions.assertThat(page.pageToken.pageSize).isEqualTo(2); + Assertions.assertThat(((OffsetPageToken) page.pageToken).offset).isEqualTo(2); + Assertions.assertThat(page.data).isEqualTo(data); + + Assertions.assertThat(OffsetPageToken.builder().fromString(page.pageToken.toString())) + .isEqualTo(page.pageToken); + } + + @Test + void testEntityIdPageToken() { + EntityIdPageToken token = EntityIdPageToken.builder().fromLimit(2); + + Assertions.assertThat(token).isInstanceOf(EntityIdPageToken.class); + Assertions.assertThat(token.id).isEqualTo(-1L); + + List badData = List.of("some", "data"); + Assertions.assertThatThrownBy(() -> token.buildNextPage(badData)) + .isInstanceOf(IllegalArgumentException.class); + + List data = + List.of(ModelEntity.builder().id(101).build(), ModelEntity.builder().id(102).build()); + var page = token.buildNextPage(data); + + Assertions.assertThat(page.pageToken).isNotNull(); + Assertions.assertThat(page.pageToken).isInstanceOf(EntityIdPageToken.class); + Assertions.assertThat(page.pageToken.pageSize).isEqualTo(2); + Assertions.assertThat(((EntityIdPageToken) page.pageToken).id).isEqualTo(102); + Assertions.assertThat(page.data).isEqualTo(data); + + Assertions.assertThat(EntityIdPageToken.builder().fromString(page.pageToken.toString())) + .isEqualTo(page.pageToken); + } +} diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java index 7b3fc9e56..1be979017 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java @@ -35,7 +35,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.catalog.pagination.PageToken; +import org.apache.polaris.core.catalog.pagination.EntityIdPageToken; +import org.apache.polaris.core.catalog.pagination.ReadEverythingPageToken; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -130,7 +131,7 @@ void testCreateEntities() { null, PolarisEntityType.TASK, PolarisEntitySubType.NULL_SUBTYPE, - PageToken.readEverything()) + ReadEverythingPageToken.get()) .getEntities(); Assertions.assertThat(listedEntities) .isNotNull() @@ -292,7 +293,9 @@ void testLoadTasks() { PolarisMetaStoreManager metaStoreManager = polarisTestMetaStoreManager.polarisMetaStoreManager; PolarisCallContext callCtx = polarisTestMetaStoreManager.polarisCallContext; List taskList = - metaStoreManager.loadTasks(callCtx, executorId, PageToken.fromLimit(5)).getEntities(); + metaStoreManager + .loadTasks(callCtx, executorId, callCtx.getMetaStore().pageTokenBuilder().fromLimit(5)) + .getEntities(); Assertions.assertThat(taskList) .isNotNull() .isNotEmpty() @@ -312,7 +315,9 @@ void testLoadTasks() { // grab a second round of tasks. Assert that none of the original 5 are in the list List newTaskList = - metaStoreManager.loadTasks(callCtx, executorId, PageToken.fromLimit(5)).getEntities(); + metaStoreManager + .loadTasks(callCtx, executorId, callCtx.getMetaStore().pageTokenBuilder().fromLimit(5)) + .getEntities(); Assertions.assertThat(newTaskList) .isNotNull() .isNotEmpty() @@ -326,7 +331,9 @@ void testLoadTasks() { // only 10 tasks are unassigned. Requesting 20, we should only receive those 10 List lastTen = - metaStoreManager.loadTasks(callCtx, executorId, PageToken.fromLimit(20)).getEntities(); + metaStoreManager + .loadTasks(callCtx, executorId, callCtx.getMetaStore().pageTokenBuilder().fromLimit(20)) + .getEntities(); Assertions.assertThat(lastTen) .isNotNull() @@ -340,7 +347,9 @@ void testLoadTasks() { .collect(Collectors.toSet()); List emtpyList = - metaStoreManager.loadTasks(callCtx, executorId, PageToken.fromLimit(20)).getEntities(); + metaStoreManager + .loadTasks(callCtx, executorId, callCtx.getMetaStore().pageTokenBuilder().fromLimit(20)) + .getEntities(); Assertions.assertThat(emtpyList).isNotNull().isEmpty(); @@ -348,7 +357,9 @@ void testLoadTasks() { // all the tasks are unassigned. Fetch them all List allTasks = - metaStoreManager.loadTasks(callCtx, executorId, PageToken.fromLimit(20)).getEntities(); + metaStoreManager + .loadTasks(callCtx, executorId, callCtx.getMetaStore().pageTokenBuilder().fromLimit(20)) + .getEntities(); Assertions.assertThat(allTasks) .isNotNull() @@ -363,7 +374,9 @@ void testLoadTasks() { timeSource.add(Duration.ofMinutes(10)); List finalList = - metaStoreManager.loadTasks(callCtx, executorId, PageToken.fromLimit(20)).getEntities(); + metaStoreManager + .loadTasks(callCtx, executorId, callCtx.getMetaStore().pageTokenBuilder().fromLimit(20)) + .getEntities(); Assertions.assertThat(finalList).isNotNull().isEmpty(); } @@ -394,7 +407,8 @@ void testLoadTasksInParallel() throws Exception { try { taskList = metaStoreManager - .loadTasks(callCtx, executorId, PageToken.fromLimit(5)) + .loadTasks( + callCtx, executorId, callCtx.getMetaStore().pageTokenBuilder().fromLimit(5)) .getEntities(); taskList.stream().map(PolarisBaseEntity::getName).forEach(taskNames::add); } catch (RetryOnConcurrencyException e) { diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java index 8692ee38c..401e4d40e 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java @@ -28,7 +28,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.catalog.pagination.PageToken; +import org.apache.polaris.core.catalog.pagination.ReadEverythingPageToken; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntity; @@ -655,7 +655,7 @@ void dropEntity(List catalogPath, PolarisEntityCore entityToD path, PolarisEntityType.NAMESPACE, PolarisEntitySubType.NULL_SUBTYPE, - PageToken.readEverything()) + ReadEverythingPageToken.get()) .getEntities(); Assertions.assertThat(children).isNotNull(); if (children.isEmpty() && entity.getType() == PolarisEntityType.NAMESPACE) { @@ -666,7 +666,7 @@ void dropEntity(List catalogPath, PolarisEntityCore entityToD path, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE, - PageToken.readEverything()) + ReadEverythingPageToken.get()) .getEntities(); Assertions.assertThat(children).isNotNull(); } else if (children.isEmpty()) { @@ -677,7 +677,7 @@ void dropEntity(List catalogPath, PolarisEntityCore entityToD path, PolarisEntityType.CATALOG_ROLE, PolarisEntitySubType.ANY_SUBTYPE, - PageToken.readEverything()) + ReadEverythingPageToken.get()) .getEntities(); Assertions.assertThat(children).isNotNull(); // if only one left, it can be dropped. @@ -1191,7 +1191,7 @@ private void validateListReturn( path, entityType, entitySubType, - PageToken.readEverything()) + ReadEverythingPageToken.get()) .getEntities(); Assertions.assertThat(result).isNotNull(); @@ -1505,7 +1505,7 @@ void validateBootstrap() { null, PolarisEntityType.PRINCIPAL, PolarisEntitySubType.NULL_SUBTYPE, - PageToken.readEverything()) + ReadEverythingPageToken.get()) .getEntities(); // ensure not null, one element only @@ -1532,7 +1532,7 @@ void validateBootstrap() { null, PolarisEntityType.PRINCIPAL_ROLE, PolarisEntitySubType.NULL_SUBTYPE, - PageToken.readEverything()) + ReadEverythingPageToken.get()) .getEntities(); // ensure not null, one element only diff --git a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index 01cd70734..aaa91b6fb 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -56,8 +56,8 @@ import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizableOperation; import org.apache.polaris.core.auth.PolarisAuthorizer; -import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; +import org.apache.polaris.core.catalog.pagination.ReadEverythingPageToken; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.CatalogRoleEntity; @@ -723,7 +723,7 @@ private List listCatalogsUnsafe() { null, PolarisEntityType.CATALOG, PolarisEntitySubType.ANY_SUBTYPE, - entityManager.newMetaStoreSession().pageTokenBuilder().readEverything()) + ReadEverythingPageToken.get()) .getEntities() .stream() .map( @@ -902,7 +902,7 @@ public List listPrincipals() { null, PolarisEntityType.PRINCIPAL, PolarisEntitySubType.NULL_SUBTYPE, - entityManager.newMetaStoreSession().pageTokenBuilder().readEverything()) + ReadEverythingPageToken.get()) .getEntities() .stream() .map( @@ -1025,7 +1025,7 @@ public List listPrincipalRoles() { null, PolarisEntityType.PRINCIPAL_ROLE, PolarisEntitySubType.NULL_SUBTYPE, - entityManager.newMetaStoreSession().pageTokenBuilder().readEverything()) + ReadEverythingPageToken.get()) .getEntities() .stream() .map( @@ -1167,7 +1167,7 @@ public List listCatalogRoles(String catalogName) { PolarisEntity.toCoreList(List.of(catalogEntity)), PolarisEntityType.CATALOG_ROLE, PolarisEntitySubType.NULL_SUBTYPE, - entityManager.newMetaStoreSession().pageTokenBuilder().readEverything()) + ReadEverythingPageToken.get()) .getEntities() .stream() .map( diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index d19d0b123..b8de85b14 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -78,9 +78,10 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; +import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.catalog.pagination.PolarisPage; +import org.apache.polaris.core.catalog.pagination.ReadEverythingPageToken; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.NamespaceEntity; @@ -461,7 +462,7 @@ private boolean paginationEnabled() { @Override public List listTables(Namespace namespace) { - return listTables(namespace, pageTokenBuilder.readEverything()).data; + return listTables(namespace, ReadEverythingPageToken.get()).data; } public PolarisPage listTables(Namespace namespace, PageToken pageToken) { @@ -762,7 +763,7 @@ public List listNamespaces() { @Override public List listNamespaces(Namespace namespace) throws NoSuchNamespaceException { - return listNamespaces(namespace, pageTokenBuilder.readEverything()).data; + return listNamespaces(namespace, ReadEverythingPageToken.get()).data; } public PolarisPage listNamespaces(PageToken pageToken) @@ -805,7 +806,7 @@ public void close() throws IOException {} @Override public List listViews(Namespace namespace) { - return listViews(namespace, pageTokenBuilder.readEverything()).data; + return listViews(namespace, ReadEverythingPageToken.get()).data; } public PolarisPage listViews(Namespace namespace, PageToken pageToken) { @@ -1092,7 +1093,7 @@ private void validateNoLocationOverlap( parentPath.stream().map(PolarisEntity::toCore).collect(Collectors.toList()), PolarisEntityType.NAMESPACE, PolarisEntitySubType.ANY_SUBTYPE, - pageTokenBuilder.readEverything()); + ReadEverythingPageToken.get()); if (!siblingNamespacesResult.isSuccess()) { throw new IllegalStateException( "Unable to resolve siblings entities to validate location - could not list namespaces"); @@ -1119,7 +1120,7 @@ private void validateNoLocationOverlap( .collect(Collectors.toList()), PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE, - pageTokenBuilder.readEverything()); + ReadEverythingPageToken.get()); if (!siblingTablesResult.isSuccess()) { throw new IllegalStateException( "Unable to resolve siblings entities to validate location - could not list tables"); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index b749eb886..7a8c52841 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -130,8 +130,12 @@ public Response listNamespaces( PolarisEntityManager entityManager = entityManagerFactory.getOrCreateEntityManager( CallContext.getCurrentContext().getRealmContext()); - PageToken token = entityManager.newMetaStoreSession() - .pageTokenBuilder().fromString(pageToken).withPageSize(pageSize); + PageToken token = + entityManager + .newMetaStoreSession() + .pageTokenBuilder() + .fromString(pageToken) + .withPageSize(pageSize); var response = newHandlerWrapper(securityContext, prefix) .listNamespaces(namespaceOptional.orElse(Namespace.of()), token); @@ -223,8 +227,12 @@ public Response listTables( PolarisEntityManager entityManager = entityManagerFactory.getOrCreateEntityManager( CallContext.getCurrentContext().getRealmContext()); - PageToken token = entityManager.newMetaStoreSession() - .pageTokenBuilder().fromString(pageToken).withPageSize(pageSize); + PageToken token = + entityManager + .newMetaStoreSession() + .pageTokenBuilder() + .fromString(pageToken) + .withPageSize(pageSize); return Response.ok(newHandlerWrapper(securityContext, prefix).listTables(ns, token)).build(); } @@ -363,8 +371,12 @@ public Response listViews( PolarisEntityManager entityManager = entityManagerFactory.getOrCreateEntityManager( CallContext.getCurrentContext().getRealmContext()); - PageToken token = entityManager.newMetaStoreSession() - .pageTokenBuilder().fromString(pageToken).withPageSize(pageSize); + PageToken token = + entityManager + .newMetaStoreSession() + .pageTokenBuilder() + .fromString(pageToken) + .withPageSize(pageSize); return Response.ok(newHandlerWrapper(securityContext, prefix).listViews(ns, token)).build(); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java index 4fbba5b71..c6ddda523 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java @@ -71,8 +71,8 @@ import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizableOperation; import org.apache.polaris.core.auth.PolarisAuthorizer; -import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; +import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.catalog.pagination.PolarisPage; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index 86d9b2a42..bca7b426b 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -62,7 +62,6 @@ import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizer; -import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.catalog.pagination.PolarisPage; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; @@ -1266,9 +1265,8 @@ public void testPaginatedListTables() { Assertions.assertThat(catalog.listTables(NS)).isNotNull().hasSize(5); // List with a limit: - PolarisPage firstListResult = catalog.listTables( - NS, - polarisContext.getMetaStore().pageTokenBuilder().fromLimit(2)); + PolarisPage firstListResult = + catalog.listTables(NS, polarisContext.getMetaStore().pageTokenBuilder().fromLimit(2)); Assertions.assertThat(firstListResult.data.size()).isEqualTo(2); Assertions.assertThat(firstListResult.pageToken.toString()).isNotNull().isNotEmpty(); @@ -1295,7 +1293,12 @@ public void testPaginatedListViews() { } for (int i = 0; i < 5; i++) { - catalog.buildView(TableIdentifier.of(NS, "pagination_view_" + i)).create(); + catalog + .buildView(TableIdentifier.of(NS, "pagination_view_" + i)) + .withQuery("a_" + i, "SELECT 1 id") + .withSchema(SCHEMA) + .withDefaultNamespace(NS) + .create(); } try { @@ -1303,9 +1306,8 @@ public void testPaginatedListViews() { Assertions.assertThat(catalog.listViews(NS)).isNotNull().hasSize(5); // List with a limit: - PolarisPage firstListResult = catalog.listViews( - NS, - polarisContext.getMetaStore().pageTokenBuilder().fromLimit(2)); + PolarisPage firstListResult = + catalog.listViews(NS, polarisContext.getMetaStore().pageTokenBuilder().fromLimit(2)); Assertions.assertThat(firstListResult.data.size()).isEqualTo(2); Assertions.assertThat(firstListResult.pageToken.toString()).isNotNull().isNotEmpty(); @@ -1336,8 +1338,8 @@ public void testPaginatedListNamespaces() { Assertions.assertThat(catalog.listNamespaces()).isNotNull().hasSize(5); // List with a limit: - PolarisPage firstListResult = catalog.listNamespaces( - polarisContext.getMetaStore().pageTokenBuilder().fromLimit(2)); + PolarisPage firstListResult = + catalog.listNamespaces(polarisContext.getMetaStore().pageTokenBuilder().fromLimit(2)); Assertions.assertThat(firstListResult.data.size()).isEqualTo(2); Assertions.assertThat(firstListResult.pageToken.toString()).isNotNull().isNotEmpty(); @@ -1352,8 +1354,8 @@ public void testPaginatedListNamespaces() { Assertions.assertThat(finalListResult.pageToken).isNull(); // List with page size matching the amount of data - PolarisPage firstExactListResult = catalog.listNamespaces( - polarisContext.getMetaStore().pageTokenBuilder().fromLimit(5)); + PolarisPage firstExactListResult = + catalog.listNamespaces(polarisContext.getMetaStore().pageTokenBuilder().fromLimit(5)); Assertions.assertThat(firstExactListResult.data.size()).isEqualTo(5); Assertions.assertThat(firstExactListResult.pageToken.toString()).isNotNull().isNotEmpty(); @@ -1363,8 +1365,8 @@ public void testPaginatedListNamespaces() { Assertions.assertThat(secondExactListResult.pageToken).isNull(); // List with huge page size: - PolarisPage bigListResult = catalog.listNamespaces( - polarisContext.getMetaStore().pageTokenBuilder().fromLimit(9999)); + PolarisPage bigListResult = + catalog.listNamespaces(polarisContext.getMetaStore().pageTokenBuilder().fromLimit(9999)); Assertions.assertThat(bigListResult.data.size()).isEqualTo(5); Assertions.assertThat(bigListResult.pageToken).isNull(); } finally { diff --git a/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java b/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java index 448e91353..b032feb77 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java @@ -33,7 +33,6 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.catalog.pagination.OffsetPageToken; -import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.AsyncTaskType; @@ -93,10 +92,7 @@ public void testTableCleanup() throws IOException { assertThat( metaStoreManagerFactory .getOrCreateMetaStoreManager(realmContext) - .loadTasks( - polarisCallContext, - "test", - new OffsetPageToken.OffsetPageTokenBuilder().fromLimit(1)) + .loadTasks(polarisCallContext, "test", OffsetPageToken.builder().fromLimit(1)) .getEntities()) .hasSize(1) .satisfiesExactly( @@ -171,10 +167,7 @@ public void close() { assertThat( metaStoreManagerFactory .getOrCreateMetaStoreManager(realmContext) - .loadTasks( - polarisCallContext, - "test", - new OffsetPageToken.OffsetPageTokenBuilder().fromLimit(5)) + .loadTasks(polarisCallContext, "test", OffsetPageToken.builder().fromLimit(5)) .getEntities()) .hasSize(1); } @@ -244,10 +237,7 @@ public void close() { assertThat( metaStoreManagerFactory .getOrCreateMetaStoreManager(realmContext) - .loadTasks( - polarisCallContext, - "test", - new OffsetPageToken.OffsetPageTokenBuilder().fromLimit(5)) + .loadTasks(polarisCallContext, "test", OffsetPageToken.builder().fromLimit(5)) .getEntities()) .hasSize(2) .satisfiesExactly( @@ -337,10 +327,7 @@ public void testTableCleanupMultipleSnapshots() throws IOException { assertThat( metaStoreManagerFactory .getOrCreateMetaStoreManager(realmContext) - .loadTasks( - polarisCallContext, - "test", - new OffsetPageToken.OffsetPageTokenBuilder().fromLimit(5)) + .loadTasks(polarisCallContext, "test", OffsetPageToken.builder().fromLimit(5)) .getEntities()) // all three manifests should be present, even though one is excluded from the latest // snapshot From 3fc5939a030dd48f535a200ebb9b97ca1a83417e Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 9 Sep 2024 22:05:37 -0400 Subject: [PATCH 21/32] close, issue with loadtasks --- .../catalog/pagination/EntityIdPageToken.java | 2 +- .../catalog/pagination/OffsetPageToken.java | 2 +- .../core/catalog/pagination/PageToken.java | 17 ++++++--- .../pagination/ReadEverythingPageToken.java | 8 +++-- .../catalog/pagination/PageTokenTest.java | 12 ++++--- .../BasePolarisMetaStoreManagerTest.java | 5 +-- .../catalog/IcebergCatalogAdapter.java | 35 +++++++++---------- 7 files changed, 48 insertions(+), 33 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java index 18a3c6fca..4535c08b3 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java @@ -70,7 +70,7 @@ protected EntityIdPageToken fromStringComponents(List components) { } @Override - public EntityIdPageToken fromLimit(int limit) { + protected EntityIdPageToken fromLimitImpl(int limit) { return new EntityIdPageToken(BASE_ID, limit); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java index 0ef83f7e9..9aa9004a1 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java @@ -83,7 +83,7 @@ protected OffsetPageToken fromStringComponents(List components) { } @Override - public OffsetPageToken fromLimit(int limit) { + protected OffsetPageToken fromLimitImpl(int limit) { return new OffsetPageToken(BASE_OFFSET, limit); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java index b38eafe67..446bd36c5 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java @@ -72,7 +72,7 @@ public abstract static class PageTokenBuilder { /** Deserialize a string into a `PageToken` */ public final PageToken fromString(String tokenString) { if (tokenString == null) { - return ReadEverythingPageToken.get(); + throw new IllegalArgumentException("Cannot build page token from null string"); } else if (tokenString.isEmpty()) { if (this instanceof ReadEverythingPageToken.ReadEverythingPageTokenBuilder) { return ReadEverythingPageToken.get(); @@ -99,15 +99,24 @@ public final PageToken fromString(String tokenString) { } } + /** Construct a `PageToken` from a plain limit */ + public final PageToken fromLimit(Integer limit) { + if (limit == null) { + return ReadEverythingPageToken.get(); + } else { + return fromLimitImpl(limit); + } + } + + /** Construct a `PageToken` from a plain limit */ + protected abstract T fromLimitImpl(int limit); + /** * PageTokenBuilder implementations should implement this to build a PageToken from components * in a string token. These components should be the same ones returned by `getComponents` and * won't include the token prefix or the checksum. */ protected abstract T fromStringComponents(List components); - - /** Construct a `PageToken` from a plain limit */ - public abstract T fromLimit(int limit); } /** Convert this PageToken to components that the serialized token string will be built from. */ diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java index dbfa3b4a7..0ec1d1438 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java @@ -65,7 +65,7 @@ protected ReadEverythingPageToken fromStringComponents(List components) } @Override - public ReadEverythingPageToken fromLimit(int limit) { + protected ReadEverythingPageToken fromLimitImpl(int limit) { throw new UnsupportedOperationException(); } } @@ -82,6 +82,10 @@ public PageToken updated(List newData) { @Override public PageToken withPageSize(Integer pageSize) { - throw new UnsupportedOperationException(); + if (pageSize == null) { + return ReadEverythingPageToken.get(); + } else { + throw new UnsupportedOperationException(); + } } } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/catalog/pagination/PageTokenTest.java b/polaris-core/src/test/java/org/apache/polaris/core/catalog/pagination/PageTokenTest.java index d6ed225fa..648b74114 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/catalog/pagination/PageTokenTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/catalog/pagination/PageTokenTest.java @@ -69,18 +69,20 @@ void testInvalidLimits(PageToken.PageTokenBuilder builder) { Assertions.assertThatThrownBy(() -> builder.fromLimit(limit)) .isInstanceOf(IllegalArgumentException.class); } + + Assertions.assertThat(builder.fromLimit(null)).isInstanceOf(ReadEverythingPageToken.class); } @ParameterizedTest @MethodSource("getPageTokenBuilders") void testStartingTokens(PageToken.PageTokenBuilder builder) { - Assertions.assertThat(builder.fromString(null)).isNotNull(); - Assertions.assertThat(builder.fromString(null)).isEqualTo(ReadEverythingPageToken.get()); - Assertions.assertThat(builder.fromString("")).isNotNull(); if (!(builder instanceof ReadEverythingPageToken.ReadEverythingPageTokenBuilder)) { Assertions.assertThat(builder.fromString("")).isNotEqualTo(ReadEverythingPageToken.get()); } + + Assertions.assertThatThrownBy(() -> builder.fromString(null)) + .isInstanceOf(IllegalArgumentException.class); } @ParameterizedTest @@ -156,7 +158,7 @@ void testReadEverythingPageToken() { @Test void testOffsetPageToken() { - OffsetPageToken token = OffsetPageToken.builder().fromLimit(2); + OffsetPageToken token = (OffsetPageToken) OffsetPageToken.builder().fromLimit(2); Assertions.assertThat(token).isInstanceOf(OffsetPageToken.class); Assertions.assertThat(token.offset).isEqualTo(0); @@ -175,7 +177,7 @@ void testOffsetPageToken() { @Test void testEntityIdPageToken() { - EntityIdPageToken token = EntityIdPageToken.builder().fromLimit(2); + EntityIdPageToken token = (EntityIdPageToken) EntityIdPageToken.builder().fromLimit(2); Assertions.assertThat(token).isInstanceOf(EntityIdPageToken.class); Assertions.assertThat(token.id).isEqualTo(-1L); diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java index 1be979017..772776ccf 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java @@ -35,7 +35,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.catalog.pagination.EntityIdPageToken; import org.apache.polaris.core.catalog.pagination.ReadEverythingPageToken; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.AsyncTaskType; @@ -408,7 +407,9 @@ void testLoadTasksInParallel() throws Exception { taskList = metaStoreManager .loadTasks( - callCtx, executorId, callCtx.getMetaStore().pageTokenBuilder().fromLimit(5)) + callCtx, + executorId, + callCtx.getMetaStore().pageTokenBuilder().fromLimit(5)) .getEntities(); taskList.stream().map(PolarisBaseEntity::getName).forEach(taskNames::add); } catch (RetryOnConcurrencyException e) { diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 7a8c52841..cd3b97551 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -107,6 +107,19 @@ private PolarisCatalogHandlerWrapper newHandlerWrapper( polarisAuthorizer); } + private PageToken buildPageToken( + PolarisEntityManager entityManager, String tokenString, Integer pageSize) { + if (tokenString != null) { + return entityManager + .newMetaStoreSession() + .pageTokenBuilder() + .fromString(tokenString) + .withPageSize(pageSize); + } else { + return entityManager.newMetaStoreSession().pageTokenBuilder().fromLimit(pageSize); + } + } + @Override public Response createNamespace( String prefix, @@ -130,12 +143,8 @@ public Response listNamespaces( PolarisEntityManager entityManager = entityManagerFactory.getOrCreateEntityManager( CallContext.getCurrentContext().getRealmContext()); - PageToken token = - entityManager - .newMetaStoreSession() - .pageTokenBuilder() - .fromString(pageToken) - .withPageSize(pageSize); + PageToken token = buildPageToken(entityManager, pageToken, pageSize); + var response = newHandlerWrapper(securityContext, prefix) .listNamespaces(namespaceOptional.orElse(Namespace.of()), token); @@ -227,12 +236,7 @@ public Response listTables( PolarisEntityManager entityManager = entityManagerFactory.getOrCreateEntityManager( CallContext.getCurrentContext().getRealmContext()); - PageToken token = - entityManager - .newMetaStoreSession() - .pageTokenBuilder() - .fromString(pageToken) - .withPageSize(pageSize); + PageToken token = buildPageToken(entityManager, pageToken, pageSize); return Response.ok(newHandlerWrapper(securityContext, prefix).listTables(ns, token)).build(); } @@ -371,12 +375,7 @@ public Response listViews( PolarisEntityManager entityManager = entityManagerFactory.getOrCreateEntityManager( CallContext.getCurrentContext().getRealmContext()); - PageToken token = - entityManager - .newMetaStoreSession() - .pageTokenBuilder() - .fromString(pageToken) - .withPageSize(pageSize); + PageToken token = buildPageToken(entityManager, pageToken, pageSize); return Response.ok(newHandlerWrapper(securityContext, prefix).listViews(ns, token)).build(); } From 549c5ab0c6e3b63ae80c700d10f0d700f27cd5f4 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 9 Sep 2024 23:18:19 -0400 Subject: [PATCH 22/32] revert --- .../persistence/PolarisMetaStoreManagerImpl.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 04003c7fd..3a5850059 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -2025,6 +2025,14 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( String executorId, PageToken pageToken) { + long taskAgeTimeout = + callCtx + .getConfigurationStore() + .getConfiguration( + callCtx, + PolarisTaskConstants.TASK_TIMEOUT_MILLIS_CONFIG, + PolarisTaskConstants.TASK_TIMEOUT_MILLIS); + // find all available tasks PolarisPage availableTasks = ms.listActiveEntities( @@ -2036,13 +2044,6 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( entity -> { PolarisObjectMapperUtil.TaskExecutionState taskState = PolarisObjectMapperUtil.parseTaskState(entity); - long taskAgeTimeout = - callCtx - .getConfigurationStore() - .getConfiguration( - callCtx, - PolarisTaskConstants.TASK_TIMEOUT_MILLIS_CONFIG, - PolarisTaskConstants.TASK_TIMEOUT_MILLIS); return taskState == null || taskState.executor == null || callCtx.getClock().millis() - taskState.lastAttemptStartTime > taskAgeTimeout; From d5f127b8314874f461c4268b330bd4224dafb59f Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 9 Sep 2024 23:30:57 -0400 Subject: [PATCH 23/32] re introduce client side filtering --- ...olarisEclipseLinkMetaStoreSessionImpl.java | 46 +++++++++++++++---- .../eclipselink/PolarisEclipseLinkStore.java | 2 + .../PolarisMetaStoreManagerImpl.java | 2 +- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index 5c0d871de..54d82160b 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -40,6 +40,7 @@ import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -549,15 +550,42 @@ public List lookupEntityActiveBatch( @NotNull PageToken pageToken, @NotNull Predicate entityFilter, @NotNull Function transformer) { - List data = - this.store - .lookupFullEntitiesActive( - localSession.get(), catalogId, parentId, entityType, pageToken) - .stream() - .map(ModelEntity::toEntity) - .filter(entityFilter) - .map(transformer) - .collect(Collectors.toList()); + + List data; + if (entityFilter.equals(Predicates.alwaysTrue())) { + // In this case, we can push the filter down into the query + data = this.store + .lookupFullEntitiesActive( + localSession.get(), catalogId, parentId, entityType, pageToken) + .stream() + .map(ModelEntity::toEntity) + .filter(entityFilter) + .map(transformer) + .collect(Collectors.toList()); + } else { + // In this case, we cannot push the filter down into the query. We must therefore remove + // the page size limit from the PageToken and filter on the client side. + PageToken unlimitedPageSizeToken = pageToken.withPageSize(Integer.MAX_VALUE); + List rawData = this.store + .lookupFullEntitiesActive( + localSession.get(), catalogId, parentId, entityType, unlimitedPageSizeToken); + if (pageToken.pageSize < Integer.MAX_VALUE && rawData.size() > pageToken.pageSize) { + LOGGER.info( + "A page token could not be respected due to a predicate. " + + "{} records were read but the client was asked to return {}", + rawData.size(), + pageToken.pageSize); + } + + data = rawData + .stream() + .map(ModelEntity::toEntity) + .filter(entityFilter) + .limit(pageToken.pageSize) + .map(transformer) + .collect(Collectors.toList()); + } + return pageToken.buildNextPage(data); } diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java index 6d79fb506..1421d6b10 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java @@ -23,6 +23,8 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; + import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.catalog.pagination.EntityIdPageToken; import org.apache.polaris.core.catalog.pagination.PageToken; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 3a5850059..37ab94dd8 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -2018,7 +2018,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( callCtx, () -> this.loadEntity(callCtx, ms, entityCatalogId, entityId)); } - /** Refer to {@link #loadTasks(PolarisCallContext, String, int, PageToken)} */ + /** Refer to {@link #loadTasks(PolarisCallContext, String, PageToken)} */ private @NotNull EntitiesResult loadTasks( @NotNull PolarisCallContext callCtx, @NotNull PolarisMetaStoreSession ms, From 342eb8997e1e4674142f0c910352621c25786828 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 9 Sep 2024 23:32:01 -0400 Subject: [PATCH 24/32] stable tests --- .../eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index 54d82160b..518c247ca 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -572,7 +572,7 @@ public List lookupEntityActiveBatch( if (pageToken.pageSize < Integer.MAX_VALUE && rawData.size() > pageToken.pageSize) { LOGGER.info( "A page token could not be respected due to a predicate. " + - "{} records were read but the client was asked to return {}", + "{} records were read but the client was asked to return {}.", rawData.size(), pageToken.pageSize); } From af1b085e66cbf3f1f89d412b00376061464f4679 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 9 Sep 2024 23:32:05 -0400 Subject: [PATCH 25/32] lint --- ...olarisEclipseLinkMetaStoreSessionImpl.java | 40 +++++++++---------- .../eclipselink/PolarisEclipseLinkStore.java | 2 - 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index 518c247ca..bf4055a00 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -40,7 +40,6 @@ import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; -import java.util.stream.Stream; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -554,36 +553,37 @@ public List lookupEntityActiveBatch( List data; if (entityFilter.equals(Predicates.alwaysTrue())) { // In this case, we can push the filter down into the query - data = this.store - .lookupFullEntitiesActive( - localSession.get(), catalogId, parentId, entityType, pageToken) - .stream() - .map(ModelEntity::toEntity) - .filter(entityFilter) - .map(transformer) - .collect(Collectors.toList()); + data = + this.store + .lookupFullEntitiesActive( + localSession.get(), catalogId, parentId, entityType, pageToken) + .stream() + .map(ModelEntity::toEntity) + .filter(entityFilter) + .map(transformer) + .collect(Collectors.toList()); } else { // In this case, we cannot push the filter down into the query. We must therefore remove // the page size limit from the PageToken and filter on the client side. PageToken unlimitedPageSizeToken = pageToken.withPageSize(Integer.MAX_VALUE); - List rawData = this.store - .lookupFullEntitiesActive( + List rawData = + this.store.lookupFullEntitiesActive( localSession.get(), catalogId, parentId, entityType, unlimitedPageSizeToken); if (pageToken.pageSize < Integer.MAX_VALUE && rawData.size() > pageToken.pageSize) { LOGGER.info( - "A page token could not be respected due to a predicate. " + - "{} records were read but the client was asked to return {}.", + "A page token could not be respected due to a predicate. " + + "{} records were read but the client was asked to return {}.", rawData.size(), pageToken.pageSize); } - data = rawData - .stream() - .map(ModelEntity::toEntity) - .filter(entityFilter) - .limit(pageToken.pageSize) - .map(transformer) - .collect(Collectors.toList()); + data = + rawData.stream() + .map(ModelEntity::toEntity) + .filter(entityFilter) + .limit(pageToken.pageSize) + .map(transformer) + .collect(Collectors.toList()); } return pageToken.buildNextPage(data); diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java index 1421d6b10..6d79fb506 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java @@ -23,8 +23,6 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import java.util.stream.Stream; - import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.catalog.pagination.EntityIdPageToken; import org.apache.polaris.core.catalog.pagination.PageToken; From 0b313aef46adda7acbc6a9a6541d596cb7f0f797 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 9 Sep 2024 23:34:17 -0400 Subject: [PATCH 26/32] one fix --- .../polaris/core/persistence/PolarisMetaStoreManagerImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 37ab94dd8..ed7e64461 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Predicates; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; import java.util.EnumMap; @@ -1491,7 +1492,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str catalogId, PolarisEntityType.CATALOG_ROLE, ms.pageTokenBuilder().fromLimit(2), - entity -> true, + Predicates.alwaysTrue(), Function.identity()) .data; From cd74b584288c3c4f88d8ff2657c242b75d327aa9 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 10 Sep 2024 10:27:47 -0400 Subject: [PATCH 27/32] fix tests --- .../org/apache/polaris/core/PolarisConfiguration.java | 1 + .../polaris/service/catalog/BasePolarisCatalog.java | 10 ++++++---- .../service/catalog/BasePolarisCatalogTest.java | 2 ++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java index 4d85805be..a6abf559c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java @@ -179,6 +179,7 @@ public static Builder builder() { public static final PolarisConfiguration LIST_PAGINATION_ENABLED = PolarisConfiguration.builder() .key("LIST_PAGINATION_ENABLED") + .catalogConfig("list-pagination.enabled") .description("If set to true, pagination for APIs like listTables is enabled") .defaultValue(false) .build(); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 791548a35..5d9ee4053 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -448,7 +448,9 @@ private boolean paginationEnabled() { .getPolarisCallContext() .getConfigurationStore() .getConfiguration( - callContext.getPolarisCallContext(), PolarisConfiguration.LIST_PAGINATION_ENABLED); + callContext.getPolarisCallContext(), + catalogEntity, + PolarisConfiguration.LIST_PAGINATION_ENABLED); } @Override @@ -462,7 +464,7 @@ public PolarisPage listTables(Namespace namespace, PageToken pa "Cannot list tables for namespace. Namespace does not exist: %s", namespace); } if (!paginationEnabled()) { - return PolarisPage.fromData(listViews(namespace)); + pageToken = ReadEverythingPageToken.get(); } return listTableLike(PolarisEntitySubType.TABLE, namespace, pageToken); @@ -770,7 +772,7 @@ public PolarisPage listNamespaces(Namespace namespace, PageToken page throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); } if (!paginationEnabled()) { - return PolarisPage.fromData(listNamespaces(namespace)); + pageToken = ReadEverythingPageToken.get(); } List catalogPath = resolvedEntities.getRawFullPath(); @@ -807,7 +809,7 @@ public PolarisPage listViews(Namespace namespace, PageToken pag "Cannot list views for namespace. Namespace does not exist: %s", namespace); } if (!paginationEnabled()) { - return PolarisPage.fromData(listViews(namespace)); + pageToken = ReadEverythingPageToken.get(); } return listTableLike(PolarisEntitySubType.VIEW, namespace, pageToken); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index 9282394a1..35ca1ddb8 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -198,6 +198,8 @@ public void before() { PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") .addProperty( PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + PolarisConfiguration.LIST_PAGINATION_ENABLED.catalogConfig(), "true") .setStorageConfigurationInfo(storageConfigModel, storageLocation) .build()); From 778c60e5a48d064d6753604c3137f654ac2325a0 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 11 Sep 2024 12:09:27 -0400 Subject: [PATCH 28/32] autolint --- .../apache/polaris/service/catalog/BasePolarisCatalogTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index 35ca1ddb8..a33128205 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -198,8 +198,7 @@ public void before() { PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") .addProperty( PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - PolarisConfiguration.LIST_PAGINATION_ENABLED.catalogConfig(), "true") + .addProperty(PolarisConfiguration.LIST_PAGINATION_ENABLED.catalogConfig(), "true") .setStorageConfigurationInfo(storageConfigModel, storageLocation) .build()); From e82402b882824ecdace564a94df64ba087f6ba3d Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 11 Sep 2024 12:28:34 -0400 Subject: [PATCH 29/32] doc fixes --- .../eclipselink/PolarisEclipseLinkStore.java | 4 ++-- .../catalog/pagination/EntityIdPageToken.java | 18 ++++++++++++++---- .../catalog/pagination/OffsetPageToken.java | 11 ++++++++++- .../core/catalog/pagination/PolarisPage.java | 4 ++-- .../pagination/ReadEverythingPageToken.java | 7 +++++-- .../persistence/PolarisMetaStoreManager.java | 1 + .../persistence/PolarisMetaStoreSession.java | 2 +- 7 files changed, 35 insertions(+), 12 deletions(-) diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java index 6d79fb506..506dbb70d 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java @@ -290,10 +290,10 @@ List lookupFullEntitiesActive( // Currently check against ENTITIES not joining with ENTITIES_ACTIVE String hql = "SELECT m from ModelEntity m " - + " where m.catalogId=:catalogId and m.parentId=:parentId and m.typeCode=:typeCode and m.id > :tokenId"; + + "where m.catalogId=:catalogId and m.parentId=:parentId and m.typeCode=:typeCode and m.id > :tokenId"; if (pageToken instanceof EntityIdPageToken) { - hql += " ORDER BY m.id ASC"; + hql += " order by m.id asc"; } TypedQuery query = diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java index 4535c08b3..2d47da33d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java @@ -22,7 +22,12 @@ import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.persistence.models.ModelEntity; -// TODO implement and comment +/** + * A {@link PageToken} implementation that tracks the greatest ID from either + * {@link PolarisBaseEntity} or {@link ModelEntity} objects supplied in updates. + * Entities are meant to be filtered during listing such that only entities with + * and ID greater than the ID of the token are returned. + */ public class EntityIdPageToken extends PageToken { public long id; @@ -32,8 +37,11 @@ private EntityIdPageToken(long id, int pageSize) { validate(); } + /** The minimum ID that could be attached to an entity */ + private static final long MINIMUM_ID = 0; + /** The entity ID to use to start with. */ - private static final long BASE_ID = -1L; + private static final long BASE_ID = MINIMUM_ID - 1; @Override protected List getComponents() { @@ -50,6 +58,7 @@ protected PageTokenBuilder getBuilder() { return EntityIdPageToken.builder(); } + /** A {@link PageTokenBuilder} implementation for {@link EntityIdPageToken} */ public static class EntityIdPageTokenBuilder extends PageTokenBuilder { @Override @@ -80,10 +89,11 @@ public PageToken updated(List newData) { if (newData == null || newData.size() < this.pageSize) { return PageToken.DONE; } else { - if (newData.get(0) instanceof ModelEntity) { + var head = newData.get(0); + if (head instanceof ModelEntity) { return new EntityIdPageToken( ((ModelEntity) newData.get(newData.size() - 1)).getId(), this.pageSize); - } else if (newData.get(0) instanceof PolarisBaseEntity) { + } else if (head instanceof PolarisBaseEntity) { return new EntityIdPageToken( ((PolarisBaseEntity) newData.get(newData.size() - 1)).getId(), this.pageSize); } else { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java index 9aa9004a1..1ea2c4d6f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java @@ -18,9 +18,17 @@ */ package org.apache.polaris.core.catalog.pagination; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.persistence.models.ModelEntity; + import java.util.List; -/** A PageToken implementation that uses an offset to manage pagination. */ +/** + * A simple {@link PageToken} implementation that tracks the number of records + * returned. Entities are meant to be filtered during listing such that when a + * token with offset N is supplied, the first N records are omitted from the + * results. + */ public class OffsetPageToken extends PageToken { /** @@ -61,6 +69,7 @@ protected List getComponents() { return List.of(String.valueOf(this.offset), String.valueOf(this.pageSize)); } + /** A {@link PageTokenBuilder} implementation for {@link OffsetPageToken} */ public static class OffsetPageTokenBuilder extends PageTokenBuilder { private OffsetPageTokenBuilder() {} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java index 06e9ef054..cedcd1581 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java @@ -21,8 +21,8 @@ import java.util.List; /** - * A wrapper for a List of data and a PageToken that can be used to continue the listing operation - * that generated that data. + * A wrapper for a List of data and a {@link PageToken} that can be used to continue the + * listing operation that generated that data. */ public class PolarisPage { public final PageToken pageToken; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java index 0ec1d1438..cf9dc5a60 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java @@ -21,8 +21,9 @@ import java.util.List; /** - * A PageToken implementation for readers who want to read everything. The behavior when using this - * token should be the same as when reading without a token. + * A {@link PageToken} implementation for readers who want to read everything. + * The behavior when using this token should be the same as when reading + * without a token. */ public class ReadEverythingPageToken extends PageToken { @@ -31,6 +32,7 @@ private ReadEverythingPageToken() { validate(); } + /** Get a ReadEverythingPageToken */ public static ReadEverythingPageToken get() { return new ReadEverythingPageToken(); } @@ -44,6 +46,7 @@ protected PageTokenBuilder getBuilder() { return builder(); } + /** A {@link PageTokenBuilder} implementation for {@link ReadEverythingPageToken} */ public static class ReadEverythingPageTokenBuilder extends PageTokenBuilder { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java index 48cb8016b..705253583 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java @@ -314,6 +314,7 @@ class ListEntitiesResult extends BaseResult { private final List entities; private final Optional pageTokenOpt; + /** Create a {@link ListEntitiesResult} from a {@link PolarisPage} */ public static ListEntitiesResult fromPolarisPage( PolarisPage polarisPage) { return new ListEntitiesResult(polarisPage.data, Optional.ofNullable(polarisPage.pageToken)); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index 2fc233308..c2645dd69 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -533,7 +533,7 @@ boolean hasChildren( * This method is used to construct page tokens when the metastore may need them. Different * metastore implementations may bring their own PageToken implementations or share them. * - * @return A `PageToken.PageTokenBuilder` implementation compatible with this + * @return A {@link PageToken.PageTokenBuilder} implementation compatible with this * `PolarisMetaStoreSession` implementation */ @NotNull From eedf8c7aa9e61854c31876c7ca244332da6cb179 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 11 Sep 2024 12:28:39 -0400 Subject: [PATCH 30/32] autolint --- .../core/catalog/pagination/EntityIdPageToken.java | 8 ++++---- .../core/catalog/pagination/OffsetPageToken.java | 10 +++------- .../polaris/core/catalog/pagination/PolarisPage.java | 4 ++-- .../catalog/pagination/ReadEverythingPageToken.java | 5 ++--- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java index 2d47da33d..4eef4a8d2 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/EntityIdPageToken.java @@ -23,10 +23,10 @@ import org.apache.polaris.core.persistence.models.ModelEntity; /** - * A {@link PageToken} implementation that tracks the greatest ID from either - * {@link PolarisBaseEntity} or {@link ModelEntity} objects supplied in updates. - * Entities are meant to be filtered during listing such that only entities with - * and ID greater than the ID of the token are returned. + * A {@link PageToken} implementation that tracks the greatest ID from either {@link + * PolarisBaseEntity} or {@link ModelEntity} objects supplied in updates. Entities are meant to be + * filtered during listing such that only entities with and ID greater than the ID of the token are + * returned. */ public class EntityIdPageToken extends PageToken { public long id; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java index 1ea2c4d6f..151ceb900 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/OffsetPageToken.java @@ -18,16 +18,12 @@ */ package org.apache.polaris.core.catalog.pagination; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.persistence.models.ModelEntity; - import java.util.List; /** - * A simple {@link PageToken} implementation that tracks the number of records - * returned. Entities are meant to be filtered during listing such that when a - * token with offset N is supplied, the first N records are omitted from the - * results. + * A simple {@link PageToken} implementation that tracks the number of records returned. Entities + * are meant to be filtered during listing such that when a token with offset N is supplied, the + * first N records are omitted from the results. */ public class OffsetPageToken extends PageToken { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java index cedcd1581..9440735d7 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java @@ -21,8 +21,8 @@ import java.util.List; /** - * A wrapper for a List of data and a {@link PageToken} that can be used to continue the - * listing operation that generated that data. + * A wrapper for a List of data and a {@link PageToken} that can be used to continue the listing + * operation that generated that data. */ public class PolarisPage { public final PageToken pageToken; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java index cf9dc5a60..819d6acfb 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java @@ -21,9 +21,8 @@ import java.util.List; /** - * A {@link PageToken} implementation for readers who want to read everything. - * The behavior when using this token should be the same as when reading - * without a token. + * A {@link PageToken} implementation for readers who want to read everything. The behavior when + * using this token should be the same as when reading without a token. */ public class ReadEverythingPageToken extends PageToken { From 54c478f9611d33e3051f437af05f0ddedf09d560 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 11 Sep 2024 14:36:07 -0400 Subject: [PATCH 31/32] lots of doc changes --- ...olarisEclipseLinkMetaStoreSessionImpl.java | 1 + .../core/catalog/pagination/PageToken.java | 28 ++++++++++--------- .../core/catalog/pagination/PolarisPage.java | 6 ++-- .../pagination/ReadEverythingPageToken.java | 4 ++- .../service/catalog/BasePolarisCatalog.java | 1 + .../catalog/IcebergCatalogAdapter.java | 6 +++- .../ListNamespacesResponseWithPageToken.java | 6 ++-- .../ListTablesResponseWithPageToken.java | 6 ++++ 8 files changed, 37 insertions(+), 21 deletions(-) diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index bf4055a00..ab3e29f9f 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -565,6 +565,7 @@ public List lookupEntityActiveBatch( } else { // In this case, we cannot push the filter down into the query. We must therefore remove // the page size limit from the PageToken and filter on the client side. + // TODO Implement a generic predicate that can be pushed down into different metastores PageToken unlimitedPageSizeToken = pageToken.withPageSize(Integer.MAX_VALUE); List rawData = this.store.lookupFullEntitiesActive( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java index 446bd36c5..09a2a51ec 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PageToken.java @@ -69,7 +69,7 @@ public abstract static class PageTokenBuilder { */ public abstract int expectedComponents(); - /** Deserialize a string into a `PageToken` */ + /** Deserialize a string into a {@link PageToken} */ public final PageToken fromString(String tokenString) { if (tokenString == null) { throw new IllegalArgumentException("Cannot build page token from null string"); @@ -90,6 +90,7 @@ public final PageToken fromString(String tokenString) { throw new IllegalArgumentException("Invalid token format in token: " + tokenString); } + // Cut off prefix and checksum T result = fromStringComponents(Arrays.asList(parts).subList(1, parts.length - 1)); result.validate(); return result; @@ -99,7 +100,7 @@ public final PageToken fromString(String tokenString) { } } - /** Construct a `PageToken` from a plain limit */ + /** Construct a {@link PageToken} from a plain limit */ public final PageToken fromLimit(Integer limit) { if (limit == null) { return ReadEverythingPageToken.get(); @@ -108,41 +109,42 @@ public final PageToken fromLimit(Integer limit) { } } - /** Construct a `PageToken` from a plain limit */ + /** Construct a {@link PageToken} from a plain limit */ protected abstract T fromLimitImpl(int limit); /** - * PageTokenBuilder implementations should implement this to build a PageToken from components - * in a string token. These components should be the same ones returned by `getComponents` and - * won't include the token prefix or the checksum. + * {@link PageTokenBuilder} implementations should implement this to build a {@link PageToken} + * from components in a string token. These components should be the same ones returned by + * {@link #getComponents()} and won't include the token prefix or the checksum. */ protected abstract T fromStringComponents(List components); } - /** Convert this PageToken to components that the serialized token string will be built from. */ + /** Convert this into components that the serialized token string will be built from. */ protected abstract List getComponents(); /** * Builds a new page token to reflect new data that's been read. If the amount of data read is - * less than the pageSize, this will return `PageToken.DONE` (done) + * less than the pageSize, this will return {@link PageToken#DONE}(null) */ protected abstract PageToken updated(List newData); /** - * Builds a `PolarisPage` from a `List`. The `PageToken` attached to the new - * `PolarisPage` is the same as the result of calling `updated(data)` on this `PageToken`. + * Builds a {@link PolarisPage} from a {@link List}. The {@link PageToken} attached to the + * new {@link PolarisPage} is the same as the result of calling {@link #updated(List)} on this + * {@link PageToken}. */ public final PolarisPage buildNextPage(List data) { return new PolarisPage(updated(data), data); } /** - * Return a new PageToken with an updated pageSize. If the pageSize provided is null, the existing - * pageSize will be preserved. + * Return a new {@link PageToken} with an updated page size. If the pageSize provided is null, the + * existing page size will be preserved. */ public abstract PageToken withPageSize(Integer pageSize); - /** Serialize a PageToken into a string */ + /** Serialize a {@link PageToken} into a string */ @Override public final String toString() { List components = getComponents(); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java index 9440735d7..c9f537e5d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java @@ -21,8 +21,8 @@ import java.util.List; /** - * A wrapper for a List of data and a {@link PageToken} that can be used to continue the listing - * operation that generated that data. + * A wrapper for a {@link List} of data and a {@link PageToken} that can be used to continue the + * listing operation that generated that data. */ public class PolarisPage { public final PageToken pageToken; @@ -33,7 +33,7 @@ public PolarisPage(PageToken pageToken, List data) { this.data = data; } - /** Used to wrap a List of data into a PolarisPage when there is no more data */ + /** Used to wrap a {@link List} of data into a {@link PolarisPage} when there is no more data */ public static PolarisPage fromData(List data) { return new PolarisPage<>(PageToken.DONE, data); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java index 819d6acfb..5002a84c4 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/ReadEverythingPageToken.java @@ -31,7 +31,7 @@ private ReadEverythingPageToken() { validate(); } - /** Get a ReadEverythingPageToken */ + /** Get a {@link ReadEverythingPageToken} */ public static ReadEverythingPageToken get() { return new ReadEverythingPageToken(); } @@ -77,11 +77,13 @@ protected List getComponents() { return List.of(); } + /** Any time {@link ReadEverythingPageToken} is updated, everything has been read */ @Override public PageToken updated(List newData) { return PageToken.DONE; } + /** {@link ReadEverythingPageToken} does not support page size */ @Override public PageToken withPageSize(Integer pageSize) { if (pageSize == null) { diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 5d9ee4053..271d59222 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -443,6 +443,7 @@ public boolean dropTable(TableIdentifier tableIdentifier, boolean purge) { return true; } + /** Check whether pagination is enabled for list operations */ private boolean paginationEnabled() { return callContext .getPolarisCallContext() diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index a24f0442f..5ba46afa8 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import edu.umd.cs.findbugs.annotations.Nullable; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; import java.net.URLEncoder; @@ -109,8 +110,11 @@ private PolarisCatalogHandlerWrapper newHandlerWrapper( polarisAuthorizer); } + /** Build a {@link PageToken} from a string and page size. */ private PageToken buildPageToken( - PolarisEntityManager entityManager, String tokenString, Integer pageSize) { + PolarisEntityManager entityManager, + @Nullable String tokenString, + @Nullable Integer pageSize) { if (tokenString != null) { return entityManager .newMetaStoreSession() diff --git a/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java b/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java index 27986210f..c891a53da 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java @@ -28,9 +28,9 @@ import org.apache.polaris.core.catalog.pagination.PolarisPage; /** - * Used in lieu of `ListNamespacesResponse` when there may be a `PageToken` associated with the - * response. Callers can use this `PageToken` to continue the listing operation and obtain more - * results. + * Used in lieu of {@link ListNamespacesResponse} when there may be a {@link PageToken} + * associated with the response. Callers can use this {@link PageToken} to continue the + * listing operation and obtain more results. */ public class ListNamespacesResponseWithPageToken extends ListNamespacesResponse { private final PageToken pageToken; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java b/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java index 7b6dcc3f0..9bb7404ac 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java @@ -23,10 +23,16 @@ import com.google.common.base.Preconditions; import java.util.List; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.responses.ListNamespacesResponse; import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.catalog.pagination.PolarisPage; +/** + * Used in lieu of {@link ListTablesResponse} when there may be a {@link PageToken} + * associated with the response. Callers can use this {@link PageToken} to continue the + * listing operation and obtain more results. + */ public class ListTablesResponseWithPageToken extends ListTablesResponse { private final PageToken pageToken; From 2e08f0de032f831523f68db0d8a8f40d13f8e4f4 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 11 Sep 2024 14:36:11 -0400 Subject: [PATCH 32/32] autolint --- .../polaris/core/catalog/pagination/PolarisPage.java | 4 +++- .../service/types/ListNamespacesResponseWithPageToken.java | 6 +++--- .../service/types/ListTablesResponseWithPageToken.java | 7 +++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java index c9f537e5d..f0f673934 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/catalog/pagination/PolarisPage.java @@ -33,7 +33,9 @@ public PolarisPage(PageToken pageToken, List data) { this.data = data; } - /** Used to wrap a {@link List} of data into a {@link PolarisPage} when there is no more data */ + /** + * Used to wrap a {@link List} of data into a {@link PolarisPage} when there is no more data + */ public static PolarisPage fromData(List data) { return new PolarisPage<>(PageToken.DONE, data); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java b/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java index c891a53da..5e9e9d031 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/types/ListNamespacesResponseWithPageToken.java @@ -28,9 +28,9 @@ import org.apache.polaris.core.catalog.pagination.PolarisPage; /** - * Used in lieu of {@link ListNamespacesResponse} when there may be a {@link PageToken} - * associated with the response. Callers can use this {@link PageToken} to continue the - * listing operation and obtain more results. + * Used in lieu of {@link ListNamespacesResponse} when there may be a {@link PageToken} associated + * with the response. Callers can use this {@link PageToken} to continue the listing operation and + * obtain more results. */ public class ListNamespacesResponseWithPageToken extends ListNamespacesResponse { private final PageToken pageToken; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java b/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java index 9bb7404ac..7689c2576 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/types/ListTablesResponseWithPageToken.java @@ -23,15 +23,14 @@ import com.google.common.base.Preconditions; import java.util.List; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.rest.responses.ListNamespacesResponse; import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.polaris.core.catalog.pagination.PageToken; import org.apache.polaris.core.catalog.pagination.PolarisPage; /** - * Used in lieu of {@link ListTablesResponse} when there may be a {@link PageToken} - * associated with the response. Callers can use this {@link PageToken} to continue the - * listing operation and obtain more results. + * Used in lieu of {@link ListTablesResponse} when there may be a {@link PageToken} associated with + * the response. Callers can use this {@link PageToken} to continue the listing operation and obtain + * more results. */ public class ListTablesResponseWithPageToken extends ListTablesResponse { private final PageToken pageToken;