From 002f415afb28d587ee704a02b63d19716a3e8618 Mon Sep 17 00:00:00 2001 From: Damir Abdullin Date: Thu, 3 Aug 2023 15:45:48 +0200 Subject: [PATCH] Data deprecation & metadata stale (#1399) Co-authored-by: nikitadementev Co-authored-by: alexeykozyurov --- .../auth/util/SecurityConstants.java | 12 +- .../controller/DataEntityController.java | 18 +- .../oddplatform/dto/DataEntityStatusDto.java | 36 ++ .../oddplatform/dto/FacetType.java | 3 +- .../dto/activity/ActivityEventTypeDto.java | 2 +- .../activity/DataEntityStatusUpdatedDto.java | 8 + .../dto/ingestion/DataEntityIngestionDto.java | 4 +- .../EnrichedDataEntityIngestionDto.java | 16 +- .../dto/policy/PolicyPermissionDto.java | 2 +- .../config/HousekeepingTTLProperties.java | 1 + .../job/AlertHousekeepingJob.java | 3 +- .../job/DataEntityHousekeepingJob.java | 386 ++++++++++++++++++ .../oddplatform/mapper/ActivityMapper.java | 18 +- .../oddplatform/mapper/DataEntityMapper.java | 10 +- .../mapper/DataEntityMapperImpl.java | 83 ++-- .../mapper/DataEntityStatusMapper.java | 22 + .../mapper/FacetStateMapperImpl.java | 3 +- .../oddplatform/mapper/LineageMapper.java | 26 ++ .../mapper/PrometheusMetricsMapperImpl.java | 2 - .../mapper/ingestion/IngestionMapperImpl.java | 47 ++- .../AlertNotificationMessageTranslator.java | 4 +- .../AlertHaltConfigRepositoryImpl.java | 3 +- .../reactive/ReactiveAlertRepositoryImpl.java | 11 +- ...eactiveDataEntityFilledRepositoryImpl.java | 7 +- .../ReactiveDataEntityRepository.java | 24 +- .../ReactiveDataEntityRepositoryImpl.java | 128 ++++-- ...activeDataEntityTaskRunRepositoryImpl.java | 1 - .../ReactiveDataQualityRepositoryImpl.java | 29 +- .../ReactiveDatasetVersionRepositoryImpl.java | 4 +- ...ReactiveGroupEntityRelationRepository.java | 8 +- ...tiveGroupEntityRelationRepositoryImpl.java | 62 +-- ...iveGroupParentGroupRelationRepository.java | 5 + ...roupParentGroupRelationRepositoryImpl.java | 23 ++ .../reactive/ReactiveLineageRepository.java | 7 +- .../ReactiveLineageRepositoryImpl.java | 48 ++- .../reactive/ReactiveOwnershipRepository.java | 2 - .../ReactiveOwnershipRepositoryImpl.java | 9 - ...eactiveSearchEntrypointRepositoryImpl.java | 15 +- .../ReactiveSearchFacetRepository.java | 5 + .../ReactiveSearchFacetRepositoryImpl.java | 117 +++++- .../reactive/ReactiveTagRepository.java | 2 - .../reactive/ReactiveTagRepositoryImpl.java | 9 - .../reactive/ReactiveTermRepositoryImpl.java | 20 +- .../reactive/TermRelationsRepository.java | 2 - .../reactive/TermRelationsRepositoryImpl.java | 9 - .../util/DataEntityCTEQueryConfig.java | 1 + .../repository/util/FTSConstants.java | 3 +- .../repository/util/JooqFTSHelper.java | 5 +- .../service/AlertHaltConfigServiceImpl.java | 2 +- .../oddplatform/service/AlertServiceImpl.java | 2 +- .../service/DataEntityFilledServiceImpl.java | 3 +- .../service/DataEntityGroupService.java | 3 - .../service/DataEntityGroupServiceImpl.java | 37 +- ...aEntityInternalInformationServiceImpl.java | 44 -- ...va => DataEntityInternalStateService.java} | 9 +- .../DataEntityInternalStateServiceImpl.java | 188 +++++++++ .../service/DataEntityRelationsService.java | 9 + .../DataEntityRelationsServiceImpl.java | 40 ++ .../service/DataEntityService.java | 6 +- .../service/DataEntityServiceImpl.java | 74 ++-- .../service/DataEntityStaleDetector.java | 18 + .../service/DataEntityStatisticsService.java | 2 + .../DataEntityStatisticsServiceImpl.java | 5 + .../service/DataQualityServiceImpl.java | 12 +- .../service/DataSourceServiceImpl.java | 2 +- .../service/NamespaceServiceImpl.java | 3 +- .../service/OwnershipServiceImpl.java | 2 +- .../oddplatform/service/TagService.java | 3 - .../oddplatform/service/TagServiceImpl.java | 6 - .../service/activity/ActivityServiceImpl.java | 8 +- .../CustomGroupDeletedActivityHandler.java | 35 -- ...ataEntityStatusUpdatedActivityHandler.java | 64 +++ .../service/attachment/FileUploadService.java | 3 + .../local/LocalFileUploadServiceImpl.java | 11 + .../remote/RemoteFileUploadServiceImpl.java | 10 + .../ingestion/IngestionServiceImpl.java | 37 +- ...setStructureIngestionRequestProcessor.java | 1 - .../job/DataEntityStatusSwitchJob.java | 32 ++ .../service/search/SearchServiceImpl.java | 1 + .../utils/ActivityParameterNames.java | 8 +- .../src/main/resources/application.yml | 2 + .../migration/V0_0_79__data_deprecation.sql | 22 + .../db/migration/V0_0_80__metadata_stale.sql | 37 ++ .../main/resources/schema/policy_schema.json | 4 +- .../oddplatform/BaseIngestionTest.java | 8 +- .../api/DataEntityStatusChangeTest.java | 90 ++++ .../ingestion/DataEntityStatisticsTest.java | 28 +- .../api/ingestion/LineageIngestionTest.java | 52 ++- .../ingestion/utils/IngestionModelMapper.java | 6 +- .../mapper/ActivityMapperTest.java | 24 +- .../mapper/DatasetVersionMapperTest.java | 2 +- .../oddplatform/mapper/LineageMapperTest.java | 12 +- .../DataQualityRepositoryImplTest.java | 19 +- .../repository/LineageRepositoryTest.java | 146 ++++--- .../service/DataEntityServiceTest.java | 15 +- .../service/LineageServiceTest.java | 29 +- .../service/NamespaceServiceImplTest.java | 20 +- .../integration/DataEntityDomainsTest.java | 154 +++++++ .../policy/valid/combined_policy.json | 3 +- odd-platform-specification/components.yaml | 74 +++- odd-platform-specification/openapi.yaml | 34 +- .../ActivityItem/ActivityItem.tsx | 17 +- .../ActivityItem/ActivityItem.tsx | 15 +- .../DataEntityDetails/DataEntityDetails.tsx | 13 +- .../DataEntityDetailsHeader.tsx | 116 +++--- .../DataEntityDetailsRoutes.tsx | 73 +++- .../DataEntityDetailsTabs.tsx | 16 +- .../DataEntityGroupControls.tsx | 91 ----- .../DatasetFieldDescription.tsx | 28 +- .../DatasetFieldOverview.tsx | 60 +-- .../DatasetFieldOverviewEnums.tsx | 44 +- .../DatasetFieldTerms/DatasetFieldTerms.tsx | 38 +- .../LinkedItemsList/LinkedItem/LinkedItem.tsx | 40 +- .../LinkedItemsList/LinkedItemsList.tsx | 4 +- .../AttachmentsHeader/AttachmentsHeader.tsx | 24 +- .../InternalDescription.tsx | 8 +- .../InternalDescriptionHeader.tsx | 18 +- .../InternalDescriptionPreview.tsx | 14 +- .../OverviewGeneral/OverviewGeneral.tsx | 29 +- .../OwnersSection/OwnersSection.tsx | 75 ++-- .../OwnershipForm/OwnershipForm.tsx | 2 +- .../OverviewGroups/OverviewGroups.tsx | 35 +- .../OverviewMetadata/OverviewMetadata.tsx | 48 ++- .../EntityGroupItem/EntityGroupItem.tsx | 21 +- .../OverviewEntityGroupItems.tsx | 6 +- .../Overview/OverviewTags/OverviewTags.tsx | 36 +- .../Overview/OverviewTerms/OverviewTerms.tsx | 34 +- .../OverviewTerms/TermItem/TermItem.tsx | 9 +- .../DataSourceList/DataSourceList.tsx | 6 + .../Directory/Directory/Directory.tsx | 13 +- .../Entities/EntitiesList/EntitiesList.tsx | 8 +- .../EntitiesList/EntityItem/EntityItem.tsx | 30 +- .../EntitiesList/TableHeader/TableHeader.tsx | 4 +- .../DataEntityList/DataEntityList.tsx | 15 +- .../DataEntityList/DataEntityListStyles.ts | 20 +- .../src/components/Search/Filters/Filters.tsx | 1 + .../Search/Results/ResultItem/ResultItem.tsx | 76 ++-- .../Results/ResultItem/ResultItemStyles.ts | 1 + .../Search/Results/Results.styles.ts | 139 +++++++ .../src/components/Search/Results/Results.tsx | 2 +- .../Search/Results/ResultsStyles.ts | 77 ---- .../SearchResultsSkeleton.tsx | 2 +- .../Results/TableHeader/TableHeader.tsx | 11 +- .../LinkedItem/LinkedItem.tsx | 41 +- .../TermLinkedItemsList/LinkedItemsList.tsx | 4 +- .../EntityStatusActivityField.tsx | 72 ++++ .../shared/elements/Activity/index.ts | 1 + .../elements/AppDatePicker/AppDatePicker.tsx | 1 - .../AppDateTimePicker/AppDateTimePicker.tsx | 92 +++++ .../AppDateTimePicker/style.overrides.ts | 72 ++++ .../elements/AppMenuItem/AppMenuItemStyles.ts | 1 + .../DefaultEntityStatus.tsx | 80 ++++ .../EntityStatus/EntityStatus.styles.ts | 41 ++ .../elements/EntityStatus/EntityStatus.tsx | 26 ++ .../SelectableEntityStatus.tsx | 87 ++++ .../Option/Option.styles.ts | 18 + .../StatusSettingsForm/Option/Option.tsx | 16 + .../StatusSettingsForm/StatusSettingsForm.tsx | 215 ++++++++++ .../elements/MetadataStale/MetadataStale.tsx | 34 ++ .../elements/TruncatedCell/TruncatedCell.tsx | 58 +-- .../TruncatedCellMenu/TruncatedCellMenu.tsx | 58 ++- .../src/components/shared/elements/index.ts | 3 + .../src/components/shared/icons/StaleIcon.tsx | 24 ++ .../src/components/shared/icons/index.ts | 1 + odd-platform-ui/src/lib/helpers.ts | 5 + .../src/lib/hooks/api/dataEntity.ts | 22 +- .../src/lib/hooks/useAppDateTime.ts | 13 +- .../redux/selectors/dataentity.selectors.ts | 8 + .../src/redux/slices/dataentities.slice.ts | 22 +- .../src/redux/thunks/dataentities.thunks.ts | 18 - odd-platform-ui/src/theme/interfaces.ts | 5 + odd-platform-ui/src/theme/palette.ts | 32 ++ 172 files changed, 3909 insertions(+), 1206 deletions(-) create mode 100644 odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/DataEntityStatusDto.java create mode 100644 odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/activity/DataEntityStatusUpdatedDto.java create mode 100644 odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/housekeeping/job/DataEntityHousekeepingJob.java create mode 100644 odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityStatusMapper.java delete mode 100644 odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityInternalInformationServiceImpl.java rename odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/{DataEntityInternalInformationService.java => DataEntityInternalStateService.java} (50%) create mode 100644 odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityInternalStateServiceImpl.java create mode 100644 odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityRelationsService.java create mode 100644 odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityRelationsServiceImpl.java create mode 100644 odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityStaleDetector.java delete mode 100644 odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/activity/handler/CustomGroupDeletedActivityHandler.java create mode 100644 odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/activity/handler/DataEntityStatusUpdatedActivityHandler.java create mode 100644 odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/job/DataEntityStatusSwitchJob.java create mode 100644 odd-platform-api/src/main/resources/db/migration/V0_0_79__data_deprecation.sql create mode 100644 odd-platform-api/src/main/resources/db/migration/V0_0_80__metadata_stale.sql create mode 100644 odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/DataEntityStatusChangeTest.java create mode 100644 odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/integration/DataEntityDomainsTest.java delete mode 100644 odd-platform-ui/src/components/DataEntityDetails/DataEntityGroup/DataEntityGroupControls/DataEntityGroupControls.tsx create mode 100644 odd-platform-ui/src/components/Search/Results/Results.styles.ts delete mode 100644 odd-platform-ui/src/components/Search/Results/ResultsStyles.ts create mode 100644 odd-platform-ui/src/components/shared/elements/Activity/ActivityFields/EntityStatusActivityField/EntityStatusActivityField.tsx create mode 100644 odd-platform-ui/src/components/shared/elements/AppDateTimePicker/AppDateTimePicker.tsx create mode 100644 odd-platform-ui/src/components/shared/elements/AppDateTimePicker/style.overrides.ts create mode 100644 odd-platform-ui/src/components/shared/elements/EntityStatus/DefaultEntityStatus/DefaultEntityStatus.tsx create mode 100644 odd-platform-ui/src/components/shared/elements/EntityStatus/EntityStatus.styles.ts create mode 100644 odd-platform-ui/src/components/shared/elements/EntityStatus/EntityStatus.tsx create mode 100644 odd-platform-ui/src/components/shared/elements/EntityStatus/SelectableEntityStatus/SelectableEntityStatus.tsx create mode 100644 odd-platform-ui/src/components/shared/elements/EntityStatus/StatusSettingsForm/Option/Option.styles.ts create mode 100644 odd-platform-ui/src/components/shared/elements/EntityStatus/StatusSettingsForm/Option/Option.tsx create mode 100644 odd-platform-ui/src/components/shared/elements/EntityStatus/StatusSettingsForm/StatusSettingsForm.tsx create mode 100644 odd-platform-ui/src/components/shared/elements/MetadataStale/MetadataStale.tsx create mode 100644 odd-platform-ui/src/components/shared/icons/StaleIcon.tsx diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/auth/util/SecurityConstants.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/auth/util/SecurityConstants.java index 250a4cceb..57ba60b15 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/auth/util/SecurityConstants.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/auth/util/SecurityConstants.java @@ -31,12 +31,12 @@ import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_DELETE_TERM; import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_DESCRIPTION_UPDATE; import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_GROUP_CREATE; -import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_GROUP_DELETE; import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_GROUP_UPDATE; import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_INTERNAL_NAME_UPDATE; import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_OWNERSHIP_CREATE; import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_OWNERSHIP_DELETE; import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_OWNERSHIP_UPDATE; +import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_STATUS_UPDATE; import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_ENTITY_TAGS_UPDATE; import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_SOURCE_CREATE; import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_SOURCE_DELETE; @@ -243,6 +243,11 @@ DATA_ENTITY, new PathPatternParserServerWebExchangeMatcher( new PathPatternParserServerWebExchangeMatcher("/api/dataentities/{data_entity_id}/links/{link_id}", DELETE), DATA_ENTITY_ATTACHMENT_MANAGE ), + new SecurityRule( + DATA_ENTITY, + new PathPatternParserServerWebExchangeMatcher("/api/dataentities/{data_entity_id}/statuses", PUT), + DATA_ENTITY_STATUS_UPDATE + ), new SecurityRule(DATASET_FIELD, new PathPatternParserServerWebExchangeMatcher("/api/datasetfields/{dataset_field_id}/description", PUT), DATASET_FIELD_DESCRIPTION_UPDATE), @@ -269,9 +274,6 @@ DATA_ENTITY, new PathPatternParserServerWebExchangeMatcher( new SecurityRule( AuthorizationManagerType.DEG, new PathPatternParserServerWebExchangeMatcher("/api/dataentitygroups/{data_entity_group_id}", PUT), - DATA_ENTITY_GROUP_UPDATE), - new SecurityRule(AuthorizationManagerType.DEG, - new PathPatternParserServerWebExchangeMatcher("/api/dataentitygroups/{data_entity_group_id}", DELETE), - DATA_ENTITY_GROUP_DELETE) + DATA_ENTITY_GROUP_UPDATE) ); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/controller/DataEntityController.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/controller/DataEntityController.java index 80f0eb1db..06ae6fca7 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/controller/DataEntityController.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/controller/DataEntityController.java @@ -22,6 +22,8 @@ import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityLineage; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityList; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRef; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusFormData; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityTermFormData; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityUsageInfo; import org.opendatadiscovery.oddplatform.api.contract.model.InternalDescription; @@ -82,13 +84,6 @@ public Mono> createDataEntityGroup(final Mono> deleteDataEntityGroup(final Long dataEntityGroupId, - final ServerWebExchange exchange) { - return dataEntityGroupService.deleteDataEntityGroup(dataEntityGroupId) - .thenReturn(ResponseEntity.noContent().build()); - } - @Override public Mono> updateDataEntityGroup(final Long dataEntityGroupId, final Mono formData, @@ -191,6 +186,15 @@ public Mono> updateOwnership(final Long dataEntityId, .map(ResponseEntity::ok); } + @Override + public Mono> updateStatus(final Long dataEntityId, + final Mono dataEntityStatus, + final ServerWebExchange exchange) { + return dataEntityStatus + .flatMap(status -> dataEntityService.updateStatus(dataEntityId, status)) + .map(ResponseEntity::ok); + } + @Override public Mono> upsertDataEntityInternalDescription( final Long dataEntityId, diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/DataEntityStatusDto.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/DataEntityStatusDto.java new file mode 100644 index 000000000..0a5ec65e6 --- /dev/null +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/DataEntityStatusDto.java @@ -0,0 +1,36 @@ +package org.opendatadiscovery.oddplatform.dto; + +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import lombok.Getter; + +import static java.util.function.Function.identity; + +public enum DataEntityStatusDto { + UNASSIGNED(1, false), + DRAFT(2, true), + STABLE(3, false), + DEPRECATED(4, true), + DELETED(5, false); + + @Getter + private final short id; + @Getter + private final boolean isSwitchable; + + DataEntityStatusDto(final int id, + final boolean isSwitchable) { + this.id = (short) id; + this.isSwitchable = isSwitchable; + } + + private static final Map MAP = Arrays + .stream(DataEntityStatusDto.values()) + .collect(Collectors.toMap(DataEntityStatusDto::getId, identity())); + + public static Optional findById(final short id) { + return Optional.ofNullable(MAP.get(id)); + } +} diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/FacetType.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/FacetType.java index 0d3bd2ae1..5c1a3df18 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/FacetType.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/FacetType.java @@ -7,7 +7,8 @@ public enum FacetType { DATA_SOURCES, OWNERS, TAGS, - GROUPS; + GROUPS, + STATUSES; public static FacetType lookup(final String facetType) { for (final FacetType ft : FacetType.values()) { diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/activity/ActivityEventTypeDto.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/activity/ActivityEventTypeDto.java index e7de3ec87..fccea1dd1 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/activity/ActivityEventTypeDto.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/activity/ActivityEventTypeDto.java @@ -13,6 +13,7 @@ public enum ActivityEventTypeDto { TERM_ASSIGNMENT_UPDATED, DESCRIPTION_UPDATED, BUSINESS_NAME_UPDATED, + DATA_ENTITY_STATUS_UPDATED, CUSTOM_METADATA_CREATED, CUSTOM_METADATA_UPDATED, CUSTOM_METADATA_DELETED, @@ -22,7 +23,6 @@ public enum ActivityEventTypeDto { DATASET_FIELD_TERM_ASSIGNMENT_UPDATED, CUSTOM_GROUP_CREATED, CUSTOM_GROUP_UPDATED, - CUSTOM_GROUP_DELETED, ALERT_HALT_CONFIG_UPDATED, ALERT_STATUS_UPDATED, OPEN_ALERT_RECEIVED, diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/activity/DataEntityStatusUpdatedDto.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/activity/DataEntityStatusUpdatedDto.java new file mode 100644 index 000000000..8df4dfe5e --- /dev/null +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/activity/DataEntityStatusUpdatedDto.java @@ -0,0 +1,8 @@ +package org.opendatadiscovery.oddplatform.dto.activity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.time.LocalDateTime; + +public record DataEntityStatusUpdatedDto(String status, + @JsonProperty("status_switch_time") LocalDateTime statusSwitchTime) { +} diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/ingestion/DataEntityIngestionDto.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/ingestion/DataEntityIngestionDto.java index 50b2777d0..2af5640f6 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/ingestion/DataEntityIngestionDto.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/ingestion/DataEntityIngestionDto.java @@ -22,8 +22,8 @@ public class DataEntityIngestionDto { protected String oddrn; protected long dataSourceId; protected String externalDescription; - protected OffsetDateTime createdAt; - protected OffsetDateTime updatedAt; + protected OffsetDateTime sourceCreatedAt; + protected OffsetDateTime sourceUpdatedAt; protected Set entityClasses; protected DataEntityTypeDto type; protected Map metadata; diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/ingestion/EnrichedDataEntityIngestionDto.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/ingestion/EnrichedDataEntityIngestionDto.java index 6f0759f11..1e1162e8a 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/ingestion/EnrichedDataEntityIngestionDto.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/ingestion/EnrichedDataEntityIngestionDto.java @@ -2,29 +2,25 @@ import lombok.Data; import lombok.EqualsAndHashCode; +import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo; @EqualsAndHashCode(callSuper = true) @Data public class EnrichedDataEntityIngestionDto extends DataEntityIngestionDto { private long id; - private boolean updated; - + private DataEntityPojo previousVersionPojo; private Boolean datasetSchemaChanged; - public EnrichedDataEntityIngestionDto(final long id, final DataEntityIngestionDto ingestionDto) { - this(id, ingestionDto, true); - } - public EnrichedDataEntityIngestionDto(final long id, - final DataEntityIngestionDto ingestionDto, - final boolean updated) { + final DataEntityPojo previousVersionPojo, + final DataEntityIngestionDto ingestionDto) { super(ingestionDto.name, ingestionDto.oddrn, ingestionDto.dataSourceId, ingestionDto.externalDescription, - ingestionDto.createdAt, ingestionDto.updatedAt, ingestionDto.entityClasses, ingestionDto.type, + ingestionDto.sourceCreatedAt, ingestionDto.sourceUpdatedAt, ingestionDto.entityClasses, ingestionDto.type, ingestionDto.metadata, ingestionDto.tags, ingestionDto.specificAttributesJson, ingestionDto.dataSet, ingestionDto.dataTransformer, ingestionDto.dataConsumer, ingestionDto.dataQualityTest, ingestionDto.dataInput, ingestionDto.dataEntityGroup); this.id = id; - this.updated = updated; + this.previousVersionPojo = previousVersionPojo; } } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/policy/PolicyPermissionDto.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/policy/PolicyPermissionDto.java index ebfb6fe26..ed76aa21f 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/policy/PolicyPermissionDto.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/policy/PolicyPermissionDto.java @@ -33,8 +33,8 @@ public enum PolicyPermissionDto { DATASET_FIELD_DELETE_TERM(DATA_ENTITY), DATA_ENTITY_GROUP_CREATE(MANAGEMENT), DATA_ENTITY_GROUP_UPDATE(DATA_ENTITY), - DATA_ENTITY_GROUP_DELETE(DATA_ENTITY), DATA_ENTITY_ATTACHMENT_MANAGE(DATA_ENTITY), + DATA_ENTITY_STATUS_UPDATE(DATA_ENTITY), TERM_CREATE(MANAGEMENT), TERM_UPDATE(TERM), TERM_DELETE(TERM), diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/housekeeping/config/HousekeepingTTLProperties.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/housekeeping/config/HousekeepingTTLProperties.java index 6d732d3fd..c1d4dc9b6 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/housekeeping/config/HousekeepingTTLProperties.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/housekeeping/config/HousekeepingTTLProperties.java @@ -8,4 +8,5 @@ public class HousekeepingTTLProperties { private int resolvedAlertsDays; private int searchFacetsDays; + private int dataEntityDeleteDays; } \ No newline at end of file diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/housekeeping/job/AlertHousekeepingJob.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/housekeeping/job/AlertHousekeepingJob.java index 9cfb5bde6..5e0045e4f 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/housekeeping/job/AlertHousekeepingJob.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/housekeeping/job/AlertHousekeepingJob.java @@ -8,6 +8,7 @@ import org.jooq.impl.DSL; import org.opendatadiscovery.oddplatform.dto.alert.AlertStatusEnum; import org.opendatadiscovery.oddplatform.housekeeping.config.HousekeepingTTLProperties; +import org.opendatadiscovery.oddplatform.service.ingestion.util.DateTimeUtil; import org.springframework.stereotype.Component; import static org.opendatadiscovery.oddplatform.model.Tables.ALERT_CHUNK; @@ -29,7 +30,7 @@ public void doHousekeeping(final Connection connection) { .where(ALERT.STATUS.eq(AlertStatusEnum.RESOLVED.getCode())) .or(ALERT.STATUS.eq(AlertStatusEnum.RESOLVED_AUTOMATICALLY.getCode())) .and(ALERT.STATUS_UPDATED_AT.lessOrEqual( - DSL.currentLocalDateTime().minus(housekeepingTTLProperties.getResolvedAlertsDays()))) + DateTimeUtil.generateNow().minusDays(housekeepingTTLProperties.getResolvedAlertsDays()))) .fetch(ALERT.ID); dslContext.deleteFrom(ALERT_CHUNK) diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/housekeeping/job/DataEntityHousekeepingJob.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/housekeeping/job/DataEntityHousekeepingJob.java new file mode 100644 index 000000000..f6ebf3c4b --- /dev/null +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/housekeeping/job/DataEntityHousekeepingJob.java @@ -0,0 +1,386 @@ +package org.opendatadiscovery.oddplatform.housekeeping.job; + +import java.sql.Connection; +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.jooq.DSLContext; +import org.jooq.impl.DSL; +import org.opendatadiscovery.oddplatform.dto.DataEntityClassDto; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; +import org.opendatadiscovery.oddplatform.housekeeping.config.HousekeepingTTLProperties; +import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo; +import org.opendatadiscovery.oddplatform.model.tables.pojos.DatasetFieldPojo; +import org.opendatadiscovery.oddplatform.model.tables.pojos.FilePojo; +import org.opendatadiscovery.oddplatform.service.attachment.FileUploadService; +import org.opendatadiscovery.oddplatform.service.ingestion.util.DateTimeUtil; +import org.springframework.stereotype.Component; + +import static org.jooq.impl.DSL.field; +import static org.opendatadiscovery.oddplatform.model.Tables.ACTIVITY; +import static org.opendatadiscovery.oddplatform.model.Tables.ALERT; +import static org.opendatadiscovery.oddplatform.model.Tables.ALERT_CHUNK; +import static org.opendatadiscovery.oddplatform.model.Tables.ALERT_HALT_CONFIG; +import static org.opendatadiscovery.oddplatform.model.Tables.DATASET_FIELD; +import static org.opendatadiscovery.oddplatform.model.Tables.DATASET_FIELD_DESCRIPTION_UNHANDLED_TERM; +import static org.opendatadiscovery.oddplatform.model.Tables.DATASET_FIELD_METADATA_VALUE; +import static org.opendatadiscovery.oddplatform.model.Tables.DATASET_FIELD_TO_TERM; +import static org.opendatadiscovery.oddplatform.model.Tables.DATASET_STRUCTURE; +import static org.opendatadiscovery.oddplatform.model.Tables.DATASET_VERSION; +import static org.opendatadiscovery.oddplatform.model.Tables.DATA_ENTITY; +import static org.opendatadiscovery.oddplatform.model.Tables.DATA_ENTITY_DESCRIPTION_UNHANDLED_TERM; +import static org.opendatadiscovery.oddplatform.model.Tables.DATA_ENTITY_FILLED; +import static org.opendatadiscovery.oddplatform.model.Tables.DATA_ENTITY_TASK_LAST_RUN; +import static org.opendatadiscovery.oddplatform.model.Tables.DATA_ENTITY_TASK_RUN; +import static org.opendatadiscovery.oddplatform.model.Tables.DATA_ENTITY_TO_TERM; +import static org.opendatadiscovery.oddplatform.model.Tables.DATA_QUALITY_TEST_RELATIONS; +import static org.opendatadiscovery.oddplatform.model.Tables.DATA_QUALITY_TEST_SEVERITY; +import static org.opendatadiscovery.oddplatform.model.Tables.ENUM_VALUE; +import static org.opendatadiscovery.oddplatform.model.Tables.FILE; +import static org.opendatadiscovery.oddplatform.model.Tables.GROUP_ENTITY_RELATIONS; +import static org.opendatadiscovery.oddplatform.model.Tables.GROUP_PARENT_GROUP_RELATIONS; +import static org.opendatadiscovery.oddplatform.model.Tables.LABEL_TO_DATASET_FIELD; +import static org.opendatadiscovery.oddplatform.model.Tables.LINEAGE; +import static org.opendatadiscovery.oddplatform.model.Tables.LINK; +import static org.opendatadiscovery.oddplatform.model.Tables.MESSAGE; +import static org.opendatadiscovery.oddplatform.model.Tables.MESSAGE_PROVIDER_EVENT; +import static org.opendatadiscovery.oddplatform.model.Tables.METADATA_FIELD_VALUE; +import static org.opendatadiscovery.oddplatform.model.Tables.METRIC_ENTITY; +import static org.opendatadiscovery.oddplatform.model.Tables.METRIC_LABEL_VALUE; +import static org.opendatadiscovery.oddplatform.model.Tables.METRIC_POINT; +import static org.opendatadiscovery.oddplatform.model.Tables.METRIC_SERIES; +import static org.opendatadiscovery.oddplatform.model.Tables.OWNERSHIP; +import static org.opendatadiscovery.oddplatform.model.Tables.SEARCH_ENTRYPOINT; +import static org.opendatadiscovery.oddplatform.model.Tables.TAG_TO_DATA_ENTITY; + +@Component +@RequiredArgsConstructor +@Slf4j +public class DataEntityHousekeepingJob implements HousekeepingJob { + private final HousekeepingTTLProperties properties; + private final FileUploadService fileUploadService; + + @Override + public void doHousekeeping(final Connection connection) { + log.debug("Running data entity housekeeping job"); + + DSL.using(connection).transaction(ctx -> { + final DSLContext dslContext = ctx.dsl(); + final LocalDateTime deleteTime = DateTimeUtil.generateNow().minusDays(properties.getDataEntityDeleteDays()); + + final List dataEntitiesToDelete = dslContext.selectFrom(DATA_ENTITY) + .where(DATA_ENTITY.STATUS.eq(DataEntityStatusDto.DELETED.getId()) + .and(DATA_ENTITY.STATUS_UPDATED_AT.lessOrEqual(deleteTime))) + .fetch(r -> r.into(DataEntityPojo.class)); + + deleteDataEntities(dslContext, dataEntitiesToDelete); + }); + } + + private void deleteDataEntities(final DSLContext dslContext, + final List dataEntitiesToDelete) { + if (CollectionUtils.isEmpty(dataEntitiesToDelete)) { + log.debug("There are no data entities to delete"); + return; + } + + final List dataEntityIds = dataEntitiesToDelete.stream() + .map(DataEntityPojo::getId) + .toList(); + + final List dataEntityOddrns = dataEntitiesToDelete.stream() + .map(DataEntityPojo::getOddrn) + .toList(); + + deleteTagRelations(dslContext, dataEntityIds); + deleteSearchVectors(dslContext, dataEntityIds); + deleteOwnerships(dslContext, dataEntityIds); + deleteMetadata(dslContext, dataEntityIds); + deleteMessages(dslContext, dataEntityIds); + deleteMetrics(dslContext, dataEntityOddrns); + deleteLinks(dslContext, dataEntityIds); + deleteLineage(dslContext, dataEntityOddrns); + deleteGroupEntityRelations(dslContext, dataEntityOddrns); + deleteGroupParentGroupRelations(dslContext, dataEntityOddrns); + deleteTermRelations(dslContext, dataEntityIds); + deleteDataEntityFilledInfo(dslContext, dataEntityIds); + deleteDataEntityDescriptionUnhandledTerms(dslContext, dataEntityIds); + deleteAlerts(dslContext, dataEntityIds, dataEntityOddrns); + deleteActivity(dslContext, dataEntityIds); + deleteTaskRuns(dslContext, dataEntityOddrns); + deleteQTSeverity(dslContext, dataEntityIds); + deleteQTRelations(dslContext, dataEntityOddrns); + deleteFiles(dslContext, dataEntityIds); + + final List datasets = dataEntitiesToDelete.stream() + .filter(this::isDataset) + .toList(); + deleteDatasetsInformation(dslContext, datasets); + + dslContext.deleteFrom(DATA_ENTITY) + .where(DATA_ENTITY.ID.in(dataEntityIds)) + .execute(); + + log.debug("Deleted data entities with ids {}", dataEntityIds); + } + + private void deleteFiles(final DSLContext dslContext, + final List dataEntityIds) { + final List deletedFiles = dslContext.deleteFrom(FILE) + .where(FILE.DATA_ENTITY_ID.in(dataEntityIds)) + .returning() + .fetch(r -> r.into(FilePojo.class)); + + //here we get files, which were not deleted by user, so we need to delete them from storage + final List filePojos = deletedFiles.stream() + .filter(file -> file.getDeletedAt() == null) + .toList(); + fileUploadService.deleteFiles(filePojos).block(); + } + + private void deleteQTRelations(final DSLContext dslContext, + final List dataEntityOddrns) { + dslContext.deleteFrom(DATA_QUALITY_TEST_RELATIONS) + .where(DATA_QUALITY_TEST_RELATIONS.DATASET_ODDRN.in(dataEntityOddrns) + .or(DATA_QUALITY_TEST_RELATIONS.DATA_QUALITY_TEST_ODDRN.in(dataEntityOddrns))) + .execute(); + } + + private void deleteQTSeverity(final DSLContext dslContext, + final List dataEntityIds) { + dslContext.deleteFrom(DATA_QUALITY_TEST_SEVERITY) + .where(DATA_QUALITY_TEST_SEVERITY.DATASET_ID.in(dataEntityIds) + .or(DATA_QUALITY_TEST_SEVERITY.DATA_QUALITY_TEST_ID.in(dataEntityIds))) + .execute(); + } + + private void deleteTaskRuns(final DSLContext dslContext, + final List dataEntityOddrns) { + dslContext.deleteFrom(DATA_ENTITY_TASK_LAST_RUN) + .where(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN.in(dataEntityOddrns)) + .execute(); + + dslContext.deleteFrom(DATA_ENTITY_TASK_RUN) + .where(DATA_ENTITY_TASK_RUN.TASK_ODDRN.in(dataEntityOddrns)) + .execute(); + } + + private void deleteDatasetsInformation(final DSLContext dslContext, + final List datasets) { + final List datasetOddrns = datasets.stream() + .map(DataEntityPojo::getOddrn) + .toList(); + + final List datasetFields = dslContext.select(DATASET_FIELD.fields()) + .from(DATASET_FIELD) + .join(DATASET_STRUCTURE).on(DATASET_STRUCTURE.DATASET_FIELD_ID.eq(DATASET_FIELD.ID)) + .join(DATASET_VERSION).on(DATASET_VERSION.ID.eq(DATASET_STRUCTURE.DATASET_VERSION_ID)) + .where(DATASET_VERSION.DATASET_ODDRN.in(datasetOddrns)) + .fetch(r -> r.into(DatasetFieldPojo.class)); + + final List datasetFieldIds = datasetFields.stream() + .map(DatasetFieldPojo::getId) + .toList(); + + final List datasetFieldOddrns = datasetFields.stream() + .map(DatasetFieldPojo::getOddrn) + .toList(); + + deleteMetrics(dslContext, datasetFieldOddrns); + + dslContext.deleteFrom(DATASET_FIELD_DESCRIPTION_UNHANDLED_TERM) + .where(DATASET_FIELD_DESCRIPTION_UNHANDLED_TERM.DATASET_FIELD_ID.in(datasetFieldIds)) + .execute(); + + dslContext.deleteFrom(ENUM_VALUE) + .where(ENUM_VALUE.DATASET_FIELD_ID.in(datasetFieldIds)) + .execute(); + + dslContext.deleteFrom(LABEL_TO_DATASET_FIELD) + .where(LABEL_TO_DATASET_FIELD.DATASET_FIELD_ID.in(datasetFieldIds)) + .execute(); + + dslContext.deleteFrom(DATASET_FIELD_TO_TERM) + .where(DATASET_FIELD_TO_TERM.DATASET_FIELD_ID.in(datasetFieldIds)) + .execute(); + + dslContext.deleteFrom(DATASET_FIELD_METADATA_VALUE) + .where(DATASET_FIELD_METADATA_VALUE.DATASET_FIELD_ID.in(datasetFieldIds)) + .execute(); + + dslContext.deleteFrom(DATASET_STRUCTURE) + .where(DATASET_STRUCTURE.DATASET_FIELD_ID.in(datasetFieldIds)) + .execute(); + + dslContext.deleteFrom(DATASET_VERSION) + .where(DATASET_VERSION.DATASET_ODDRN.in(datasetOddrns)) + .execute(); + + dslContext.deleteFrom(DATASET_FIELD) + .where(DATASET_FIELD.ID.in(datasetFieldIds)) + .execute(); + } + + private void deleteActivity(final DSLContext dslContext, final List dataEntityIds) { + dslContext.deleteFrom(ACTIVITY) + .where(ACTIVITY.DATA_ENTITY_ID.in(dataEntityIds)) + .execute(); + } + + private void deleteAlerts(final DSLContext dslContext, + final List dataEntityIds, + final List dataEntityOddrns) { + dslContext.deleteFrom(ALERT_HALT_CONFIG) + .where(ALERT_HALT_CONFIG.DATA_ENTITY_ID.in(dataEntityIds)) + .execute(); + + final List alertIds = dslContext.selectFrom(ALERT) + .where(ALERT.DATA_ENTITY_ODDRN.in(dataEntityOddrns)) + .fetch(ALERT.ID); + + dslContext.deleteFrom(ALERT_CHUNK) + .where(ALERT_CHUNK.ALERT_ID.in(alertIds)) + .execute(); + + dslContext.deleteFrom(ALERT) + .where(ALERT.ID.in(alertIds)) + .execute(); + } + + private void deleteTagRelations(final DSLContext dslContext, + final List dataEntityIds) { + dslContext.deleteFrom(TAG_TO_DATA_ENTITY) + .where(TAG_TO_DATA_ENTITY.DATA_ENTITY_ID.in(dataEntityIds)) + .execute(); + } + + private void deleteSearchVectors(final DSLContext dslContext, + final List dataEntityIds) { + dslContext.deleteFrom(SEARCH_ENTRYPOINT) + .where(SEARCH_ENTRYPOINT.DATA_ENTITY_ID.in(dataEntityIds)) + .execute(); + } + + private void deleteOwnerships(final DSLContext dslContext, + final List dataEntityIds) { + dslContext.deleteFrom(OWNERSHIP) + .where(OWNERSHIP.DATA_ENTITY_ID.in(dataEntityIds)) + .execute(); + } + + private void deleteMetadata(final DSLContext dslContext, + final List dataEntityIds) { + dslContext.deleteFrom(METADATA_FIELD_VALUE) + .where(METADATA_FIELD_VALUE.DATA_ENTITY_ID.in(dataEntityIds)) + .execute(); + } + + private void deleteMessages(final DSLContext dslContext, + final List dataEntityIds) { + final List messageUUIDs = dslContext.select(MESSAGE.UUID) + .from(MESSAGE) + .where(MESSAGE.DATA_ENTITY_ID.in(dataEntityIds)) + .fetch(MESSAGE.UUID); + + dslContext.deleteFrom(MESSAGE_PROVIDER_EVENT) + .where(MESSAGE_PROVIDER_EVENT.PARENT_MESSAGE_UUID.in(messageUUIDs)) + .execute(); + + dslContext.deleteFrom(MESSAGE) + .where(MESSAGE.UUID.in(messageUUIDs)) + .execute(); + } + + private void deleteMetrics(final DSLContext dslContext, + final List oddrns) { + final List metricEntityIds = dslContext.select(METRIC_ENTITY.ID) + .from(METRIC_ENTITY) + .where(METRIC_ENTITY.ENTITY_ODDRN.in(oddrns)) + .fetch(METRIC_ENTITY.ID); + + final List metricSeriesIds = dslContext.select(METRIC_SERIES.ID) + .from(METRIC_SERIES) + .where(METRIC_SERIES.METRIC_ENTITY_ID.in(metricEntityIds)) + .fetch(METRIC_SERIES.ID); + + final var metricLabelValueIds = + DSL.selectDistinct(field("unnest(?)", Integer.TYPE, METRIC_POINT.LABEL_VALUES_IDS)) + .from(METRIC_POINT) + .where(METRIC_POINT.SERIES_ID.in(metricSeriesIds)); + + dslContext.deleteFrom(METRIC_LABEL_VALUE) + .where(METRIC_LABEL_VALUE.ID.in(metricLabelValueIds)) + .execute(); + + dslContext.deleteFrom(METRIC_POINT) + .where(METRIC_POINT.SERIES_ID.in(metricSeriesIds)) + .execute(); + + dslContext.deleteFrom(METRIC_SERIES) + .where(METRIC_SERIES.ID.in(metricSeriesIds)) + .execute(); + + dslContext.deleteFrom(METRIC_ENTITY) + .where(METRIC_ENTITY.ENTITY_ODDRN.in(oddrns)) + .execute(); + } + + private void deleteLinks(final DSLContext dslContext, + final List dataEntityIds) { + dslContext.deleteFrom(LINK) + .where(LINK.DATA_ENTITY_ID.in(dataEntityIds)) + .execute(); + } + + private void deleteLineage(final DSLContext dslContext, + final List dataEntityOddrns) { + dslContext.deleteFrom(LINEAGE) + .where(LINEAGE.CHILD_ODDRN.in(dataEntityOddrns).or(LINEAGE.PARENT_ODDRN.in(dataEntityOddrns))) + .execute(); + } + + private void deleteGroupEntityRelations(final DSLContext dslContext, + final List dataEntityOddrns) { + dslContext.deleteFrom(GROUP_ENTITY_RELATIONS) + .where(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN.in(dataEntityOddrns) + .or(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.in(dataEntityOddrns))) + .execute(); + } + + private void deleteGroupParentGroupRelations(final DSLContext dslContext, + final List dataEntityOddrns) { + dslContext.deleteFrom(GROUP_PARENT_GROUP_RELATIONS) + .where(GROUP_PARENT_GROUP_RELATIONS.PARENT_GROUP_ODDRN.in(dataEntityOddrns) + .or(GROUP_PARENT_GROUP_RELATIONS.GROUP_ODDRN.in(dataEntityOddrns))) + .execute(); + } + + private void deleteTermRelations(final DSLContext dslContext, final List dataEntityIds) { + dslContext.deleteFrom(DATA_ENTITY_TO_TERM) + .where(DATA_ENTITY_TO_TERM.DATA_ENTITY_ID.in(dataEntityIds)) + .execute(); + } + + private void deleteDataEntityFilledInfo(final DSLContext dslContext, + final List dataEntityIds) { + dslContext.deleteFrom(DATA_ENTITY_FILLED) + .where(DATA_ENTITY_FILLED.DATA_ENTITY_ID.in(dataEntityIds)) + .execute(); + } + + private void deleteDataEntityDescriptionUnhandledTerms(final DSLContext dslContext, + final List dataEntityIds) { + dslContext.deleteFrom(DATA_ENTITY_DESCRIPTION_UNHANDLED_TERM) + .where(DATA_ENTITY_DESCRIPTION_UNHANDLED_TERM.DATA_ENTITY_ID.in(dataEntityIds)) + .execute(); + } + + private boolean isDataset(final DataEntityPojo pojo) { + return Arrays.stream(pojo.getEntityClassIds()) + .anyMatch(classId -> DataEntityClassDto.DATA_SET.getId() == classId); + } +} diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/ActivityMapper.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/ActivityMapper.java index 6b4376e58..12c6a395a 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/ActivityMapper.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/ActivityMapper.java @@ -21,6 +21,9 @@ import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityActivityState; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityClass; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRef; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusActivityState; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusEnum; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityType; import org.opendatadiscovery.oddplatform.api.contract.model.DataSetFieldType; import org.opendatadiscovery.oddplatform.api.contract.model.DatasetFieldInformationActivityState; @@ -42,6 +45,7 @@ import org.opendatadiscovery.oddplatform.dto.activity.BusinessNameActivityStateDto; import org.opendatadiscovery.oddplatform.dto.activity.CustomGroupActivityStateDto; import org.opendatadiscovery.oddplatform.dto.activity.DataEntityCreatedActivityStateDto; +import org.opendatadiscovery.oddplatform.dto.activity.DataEntityStatusUpdatedDto; import org.opendatadiscovery.oddplatform.dto.activity.DatasetFieldInformationActivityStateDto; import org.opendatadiscovery.oddplatform.dto.activity.DatasetFieldTermsActivityStateDto; import org.opendatadiscovery.oddplatform.dto.activity.DatasetFieldValuesActivityStateDto; @@ -51,6 +55,7 @@ import org.opendatadiscovery.oddplatform.dto.activity.TermActivityStateDto; import org.opendatadiscovery.oddplatform.model.tables.pojos.ActivityPojo; import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo; +import org.opendatadiscovery.oddplatform.service.ingestion.util.DateTimeUtil; import org.opendatadiscovery.oddplatform.utils.JSONSerDeUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -113,11 +118,12 @@ ActivityState mapState(final JSONB jsonb, final ActivityEventTypeDto eventTypeDt case TERM_ASSIGNMENT_UPDATED -> mapTermsState(jsonb); case DESCRIPTION_UPDATED -> mapDescriptionState(jsonb); case BUSINESS_NAME_UPDATED -> mapBusinessNameState(jsonb); + case DATA_ENTITY_STATUS_UPDATED -> mapDataEntityStatusState(jsonb); case DATASET_FIELD_VALUES_UPDATED -> mapDatasetFieldValuesState(jsonb); case DATASET_FIELD_DESCRIPTION_UPDATED, DATASET_FIELD_LABELS_UPDATED -> mapDatasetFieldInformationState(jsonb); case DATASET_FIELD_TERM_ASSIGNMENT_UPDATED -> mapDatasetFieldTermState(jsonb); - case CUSTOM_GROUP_CREATED, CUSTOM_GROUP_UPDATED, CUSTOM_GROUP_DELETED -> mapCustomGroupState(jsonb); + case CUSTOM_GROUP_CREATED, CUSTOM_GROUP_UPDATED -> mapCustomGroupState(jsonb); case ALERT_HALT_CONFIG_UPDATED -> mapAlertHaltConfigState(jsonb); case ALERT_STATUS_UPDATED -> mapAlertUpdatedStatus(jsonb); case OPEN_ALERT_RECEIVED -> mapOpenAlertReceived(jsonb); @@ -189,6 +195,16 @@ ActivityState mapBusinessNameState(final JSONB jsonb) { abstract BusinessNameActivityState mapInternalNameActivityState(final BusinessNameActivityStateDto dto); + ActivityState mapDataEntityStatusState(final JSONB jsonb) { + final DataEntityStatusUpdatedDto stateDto = + JSONSerDeUtils.deserializeJson(jsonb.data(), DataEntityStatusUpdatedDto.class); + final DataEntityStatusActivityState state = new DataEntityStatusActivityState(); + final DataEntityStatus status = new DataEntityStatus(DataEntityStatusEnum.fromValue(stateDto.status())); + status.setStatusSwitchTime(DateTimeUtil.mapUTCDateTime(stateDto.statusSwitchTime())); + state.setStatus(status); + return new ActivityState().status(state); + } + ActivityState mapDatasetFieldValuesState(final JSONB jsonb) { final DatasetFieldValuesActivityStateDto stateDto = JSONSerDeUtils.deserializeJson(jsonb.data(), DatasetFieldValuesActivityStateDto.class); diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityMapper.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityMapper.java index 922631e7e..c6f601b17 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityMapper.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityMapper.java @@ -10,6 +10,7 @@ import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityGroupItem; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityList; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRef; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityType; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityUsageInfo; import org.opendatadiscovery.oddplatform.dto.DataEntityClassDto; @@ -30,14 +31,17 @@ public interface DataEntityMapper { DataEntityList mapPojos(final Page dataEntityDto); - DataEntityPojo mapToPojo(final DataEntityGroupFormData formData, - final DataEntityClassDto classDto, - final NamespacePojo namespacePojo); + DataEntityPojo mapCreatedDEGPojo(final DataEntityGroupFormData formData, + final DataEntityClassDto classDto, + final NamespacePojo namespacePojo); DataEntityPojo applyToPojo(final DataEntityGroupFormData formData, final NamespacePojo namespacePojo, final DataEntityPojo pojo); + DataEntityPojo applyStatus(final DataEntityPojo pojo, + final DataEntityStatus status); + DataEntityDetails mapDtoDetails(final DataEntityDetailsDto dataEntityDetailsDto); DataEntity mapDataQualityTest(final DataEntityDimensionsDto dto, final String severity); diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityMapperImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityMapperImpl.java index e49a194b5..c1167bd80 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityMapperImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityMapperImpl.java @@ -25,6 +25,7 @@ import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityList; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRef; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRun; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityType; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityTypeUsageInfo; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityUsageInfo; @@ -38,6 +39,7 @@ import org.opendatadiscovery.oddplatform.dto.DataEntityDetailsDto; import org.opendatadiscovery.oddplatform.dto.DataEntityDimensionsDto; import org.opendatadiscovery.oddplatform.dto.DataEntityDto; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; import org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto; import org.opendatadiscovery.oddplatform.dto.DataSourceDto; import org.opendatadiscovery.oddplatform.dto.attributes.LinkedUrlAttribute; @@ -45,6 +47,7 @@ import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityStatisticsPojo; import org.opendatadiscovery.oddplatform.model.tables.pojos.DataQualityTestSeverityPojo; import org.opendatadiscovery.oddplatform.model.tables.pojos.NamespacePojo; +import org.opendatadiscovery.oddplatform.service.DataEntityStaleDetector; import org.opendatadiscovery.oddplatform.service.ingestion.util.DateTimeUtil; import org.opendatadiscovery.oddplatform.utils.JSONSerDeUtils; import org.opendatadiscovery.oddplatform.utils.Page; @@ -66,6 +69,8 @@ public class DataEntityMapperImpl implements DataEntityMapper { private final DataEntityRunMapper dataEntityRunMapper; private final TermMapper termMapper; private final DateTimeMapper dateTimeMapper; + private final DataEntityStatusMapper dataEntityStatusMapper; + private final DataEntityStaleDetector dataEntityStaleDetector; @Override public DataEntity mapPojo(final DataEntityDimensionsDto dto) { @@ -148,37 +153,41 @@ private DataEntity mapPojo(final DataEntityPojo pojo) { .internalName(pojo.getInternalName()) .externalName(pojo.getExternalName()) .oddrn(pojo.getOddrn()) - .createdAt(dateTimeMapper.mapUTCDateTime(pojo.getCreatedAt())) - .updatedAt(dateTimeMapper.mapUTCDateTime(pojo.getUpdatedAt())) - .viewCount(pojo.getViewCount()); + .sourceCreatedAt(dateTimeMapper.mapUTCDateTime(pojo.getSourceCreatedAt())) + .sourceUpdatedAt(dateTimeMapper.mapUTCDateTime(pojo.getSourceUpdatedAt())) + .lastIngestedAt(dateTimeMapper.mapUTCDateTime(pojo.getLastIngestedAt())) + .viewCount(pojo.getViewCount()) + .status(dataEntityStatusMapper.mapStatus(pojo)) + .isStale(dataEntityStaleDetector.isDataEntityStale(pojo)); } @Override public DataEntityList mapPojos(final List dataEntityDto) { - return new DataEntityList() - .items(dataEntityDto.stream().map(this::mapPojo).collect(Collectors.toList())) - .pageInfo(pageInfo(dataEntityDto.size())); + final List entities = dataEntityDto.stream().map(this::mapPojo).toList(); + final PageInfo pageInfo = pageInfo(dataEntityDto.size()); + return new DataEntityList(entities, pageInfo); } @Override public DataEntityList mapPojos(final Page dataEntityDto) { - return new DataEntityList() - .items(dataEntityDto.getData().stream().map(this::mapPojo).collect(Collectors.toList())) - .pageInfo(pageInfo(dataEntityDto)); + final List entities = dataEntityDto.getData().stream().map(this::mapPojo).toList(); + final PageInfo pageInfo = pageInfo(dataEntityDto); + return new DataEntityList(entities, pageInfo); } @Override - public DataEntityPojo mapToPojo(final DataEntityGroupFormData formData, - final DataEntityClassDto classDto, - final NamespacePojo namespacePojo) { + public DataEntityPojo mapCreatedDEGPojo(final DataEntityGroupFormData formData, + final DataEntityClassDto classDto, + final NamespacePojo namespacePojo) { final LocalDateTime now = DateTimeUtil.generateNow(); return new DataEntityPojo() .setInternalName(formData.getName()) .setNamespaceId(namespacePojo != null ? namespacePojo.getId() : null) .setEntityClassIds(new Integer[] {classDto.getId()}) .setTypeId(formData.getType().getId()) - .setCreatedAt(now) - .setUpdatedAt(now) + .setPlatformCreatedAt(now) + .setStatus(DataEntityStatusDto.UNASSIGNED.getId()) + .setStatusUpdatedAt(now) .setManuallyCreated(true) .setHollow(false) .setExcludeFromSearch(false); @@ -194,8 +203,21 @@ public DataEntityPojo applyToPojo(final DataEntityGroupFormData formData, return pojo .setInternalName(formData.getName()) .setNamespaceId(namespacePojo != null ? namespacePojo.getId() : null) - .setTypeId(formData.getType().getId()) - .setUpdatedAt(DateTimeUtil.generateNow()); + .setTypeId(formData.getType().getId()); + } + + @Override + public DataEntityPojo applyStatus(final DataEntityPojo pojo, final DataEntityStatus status) { + if (pojo == null) { + return null; + } + final DataEntityStatusDto statusDto = DataEntityStatusDto.valueOf(status.getStatus().getValue()); + pojo.setStatus(statusDto.getId()); + pojo.setStatusSwitchTime(DateTimeUtil.mapUTCDateTime(status.getStatusSwitchTime())); + if (statusDto.getId() != pojo.getStatus()) { + pojo.setStatusUpdatedAt(DateTimeUtil.generateNow()); + } + return pojo; } @Override @@ -219,11 +241,14 @@ public DataEntityDetails mapDtoDetails(final DataEntityDetailsDto dto) { .oddrn(pojo.getOddrn()) .internalDescription(pojo.getInternalDescription()) .externalDescription(pojo.getExternalDescription()) - .createdAt(dateTimeMapper.mapUTCDateTime(pojo.getCreatedAt())) - .updatedAt(dateTimeMapper.mapUTCDateTime(pojo.getUpdatedAt())) + .sourceCreatedAt(dateTimeMapper.mapUTCDateTime(pojo.getSourceCreatedAt())) + .sourceUpdatedAt(dateTimeMapper.mapUTCDateTime(pojo.getSourceUpdatedAt())) + .lastIngestedAt(dateTimeMapper.mapUTCDateTime(pojo.getLastIngestedAt())) + .isStale(dataEntityStaleDetector.isDataEntityStale(pojo)) .dataEntityGroups(groups) .entityClasses(entityClasses.stream().map(this::mapEntityClass).toList()) .type(type) + .status(dataEntityStatusMapper.mapStatus(pojo)) .ownership(ownershipMapper.mapDtos(dto.getOwnership())) .dataSource(dataSourceMapper.mapDto(new DataSourceDto(dto.getDataSource(), dto.getNamespace(), null))) .tags(tagMapper.mapToTagList(dto.getTags())) @@ -365,10 +390,7 @@ public DataEntityType mapType(final DataEntityTypeDto type) { if (type == null) { return null; } - - return new DataEntityType() - .id(type.getId()) - .name(DataEntityType.NameEnum.fromValue(type.name())); + return new DataEntityType(type.getId(), DataEntityType.NameEnum.fromValue(type.name())); } @Override @@ -430,8 +452,11 @@ public DataEntityGroupItem mapGroupItem(final DataEntityDimensionsDto dimensions .ownership(ownershipMapper.mapDtos(dimensionsDto.getOwnership())) .entityClasses(entityClasses.stream().map(this::mapEntityClass).toList()) .type(type) - .createdAt(DateTimeUtil.mapUTCDateTime(pojo.getCreatedAt())) - .updatedAt(DateTimeUtil.mapUTCDateTime(pojo.getUpdatedAt())); + .status(dataEntityStatusMapper.mapStatus(pojo)) + .sourceCreatedAt(DateTimeUtil.mapUTCDateTime(pojo.getSourceCreatedAt())) + .sourceUpdatedAt(DateTimeUtil.mapUTCDateTime(pojo.getSourceUpdatedAt())) + .lastIngestedAt(DateTimeUtil.mapUTCDateTime(pojo.getLastIngestedAt())) + .isStale(dataEntityStaleDetector.isDataEntityStale(pojo)); item.setDataEntity(dataEntity); return item; } @@ -459,9 +484,7 @@ private DataEntityClassUsageInfo mapToEntityClassUsage(final DataEntityClassDto } private LinkedUrl mapLinkedUrl(final LinkedUrlAttribute linkedUrlAttribute) { - return new LinkedUrl() - .url(linkedUrlAttribute.url()) - .name(linkedUrlAttribute.name()); + return new LinkedUrl(linkedUrlAttribute.url(), linkedUrlAttribute.name()); } private List mapLinkedUrlList(final Collection linkedUrlAttributes) { @@ -487,6 +510,8 @@ private DataEntityRef mapReference(final DataEntityPojo pojo) { .internalName(pojo.getInternalName()) .entityClasses(entityClasses) .manuallyCreated(pojo.getManuallyCreated()) + .status(dataEntityStatusMapper.mapStatus(pojo)) + .isStale(dataEntityStaleDetector.isDataEntityStale(pojo)) .url(""); } @@ -517,10 +542,10 @@ private DataEntityType getDataEntityType(final DataEntityPojo pojo) { } private PageInfo pageInfo(final long total) { - return new PageInfo().total(total).hasNext(false); + return new PageInfo(total, false); } private PageInfo pageInfo(final Page page) { - return new PageInfo().total(page.getTotal()).hasNext(page.isHasNext()); + return new PageInfo(page.getTotal(), page.isHasNext()); } } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityStatusMapper.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityStatusMapper.java new file mode 100644 index 000000000..cbcdd061e --- /dev/null +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityStatusMapper.java @@ -0,0 +1,22 @@ +package org.opendatadiscovery.oddplatform.mapper; + +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusEnum; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; +import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo; +import org.opendatadiscovery.oddplatform.service.ingestion.util.DateTimeUtil; +import org.springframework.stereotype.Component; + +@Component +public class DataEntityStatusMapper { + + public DataEntityStatus mapStatus(final DataEntityPojo pojo) { + final DataEntityStatusDto statusDto = DataEntityStatusDto.findById(pojo.getStatus()) + .orElseThrow(() -> new IllegalArgumentException("Status shouldn't be null")); + final DataEntityStatus status = new DataEntityStatus(DataEntityStatusEnum.fromValue(statusDto.name())); + if (pojo.getStatusSwitchTime() != null) { + status.setStatusSwitchTime(DateTimeUtil.mapUTCDateTime(pojo.getStatusSwitchTime())); + } + return status; + } +} diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapperImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapperImpl.java index 2362ed879..ca048ff2f 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapperImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapperImpl.java @@ -43,7 +43,8 @@ public class FacetStateMapperImpl implements FacetStateMapper { SearchFormDataFilters::getNamespaces, FacetType.NAMESPACES, SearchFormDataFilters::getOwners, FacetType.OWNERS, SearchFormDataFilters::getTags, FacetType.TAGS, - SearchFormDataFilters::getGroups, FacetType.GROUPS + SearchFormDataFilters::getGroups, FacetType.GROUPS, + SearchFormDataFilters::getStatuses, FacetType.STATUSES ); private static final Map>, FacetType> diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/LineageMapper.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/LineageMapper.java index 3418d29a8..09f06904b 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/LineageMapper.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/LineageMapper.java @@ -15,6 +15,7 @@ import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityLineageEdge; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityLineageNode; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityLineageStream; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; import org.opendatadiscovery.oddplatform.api.contract.model.DataSource; import org.opendatadiscovery.oddplatform.dto.DataEntityClassDto; import org.opendatadiscovery.oddplatform.dto.DataEntityDimensionsDto; @@ -23,6 +24,7 @@ import org.opendatadiscovery.oddplatform.dto.lineage.DataEntityLineageDto; import org.opendatadiscovery.oddplatform.dto.lineage.DataEntityLineageStreamDto; import org.opendatadiscovery.oddplatform.dto.lineage.LineageNodeDto; +import org.opendatadiscovery.oddplatform.service.DataEntityStaleDetector; import org.opendatadiscovery.oddplatform.utils.Pair; import org.springframework.beans.factory.annotation.Autowired; @@ -33,6 +35,8 @@ public abstract class LineageMapper { private DataEntityMapper dataEntityMapper; private DataSourceMapper dataSourceMapper; + private DataEntityStatusMapper dataEntityStatusMapper; + private DataEntityStaleDetector dataEntityStaleDetector; @Autowired public void setDataEntityMapper(final DataEntityMapper dataEntityMapper) { @@ -44,6 +48,16 @@ public void setDataSourceMapper(final DataSourceMapper dataSourceMapper) { this.dataSourceMapper = dataSourceMapper; } + @Autowired + public void setDataEntityStatusMapper(final DataEntityStatusMapper dataEntityStatusMapper) { + this.dataEntityStatusMapper = dataEntityStatusMapper; + } + + @Autowired + public void setDataEntityStaleDetector(final DataEntityStaleDetector dataEntityStaleDetector) { + this.dataEntityStaleDetector = dataEntityStaleDetector; + } + @Mapping(target = "root", source = "dataEntityDto") public abstract DataEntityLineage mapLineageDto(final DataEntityLineageDto dataEntityLineageDto); @@ -99,6 +113,8 @@ DataEntityLineageNode mapRootOrGroupNode(final DataEntityDimensionsDto dto) { @Mapping(target = "internalName", source = "dto.entity.dataEntity.internalName") @Mapping(target = "entityClasses", source = "dto", qualifiedByName = "entityClassesNamed") @Mapping(target = "dataSource", source = "dto", qualifiedByName = "dtoToDataSourceNamed") + @Mapping(target = "status", source = "dto", qualifiedByName = "dtoToDataEntityStatusNamed") + @Mapping(target = "isStale", source = "dto", qualifiedByName = "dtoToDataEntityIsStaleNamed") abstract DataEntityLineageNode mapNode(final LineageNodeDto dto, final List groupIds); @Named("entityClassesNamed") @@ -116,5 +132,15 @@ DataSource mapDtoToDataSource(final LineageNodeDto dto) { dto.entity().getNamespace(), null)) : null; } + + @Named("dtoToDataEntityStatusNamed") + DataEntityStatus mapStatus(final LineageNodeDto dto) { + return dataEntityStatusMapper.mapStatus(dto.entity().getDataEntity()); + } + + @Named("dtoToDataEntityIsStaleNamed") + Boolean mapIsStale(final LineageNodeDto dto) { + return dataEntityStaleDetector.isDataEntityStale(dto.entity().getDataEntity()); + } } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/PrometheusMetricsMapperImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/PrometheusMetricsMapperImpl.java index 3ccf13605..061082dfc 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/PrometheusMetricsMapperImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/PrometheusMetricsMapperImpl.java @@ -1,9 +1,7 @@ package org.opendatadiscovery.oddplatform.mapper; import java.math.BigDecimal; -import java.time.Instant; import java.time.OffsetDateTime; -import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Comparator; import java.util.List; diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/ingestion/IngestionMapperImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/ingestion/IngestionMapperImpl.java index b983159d1..9ab8e267a 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/ingestion/IngestionMapperImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/ingestion/IngestionMapperImpl.java @@ -1,5 +1,6 @@ package org.opendatadiscovery.oddplatform.mapper.ingestion; +import java.time.LocalDateTime; import java.time.OffsetDateTime; import java.util.Collection; import java.util.HashSet; @@ -17,6 +18,7 @@ import org.apache.commons.lang3.StringUtils; import org.jooq.JSONB; import org.opendatadiscovery.oddplatform.dto.DataEntityClassDto; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; import org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto; import org.opendatadiscovery.oddplatform.dto.ingestion.DataEntityIngestionDto; import org.opendatadiscovery.oddplatform.dto.ingestion.EnrichedDataEntityIngestionDto; @@ -37,6 +39,7 @@ import org.opendatadiscovery.oddplatform.ingestion.contract.model.Tag; import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo; import org.opendatadiscovery.oddplatform.service.ingestion.DatasetVersionHashCalculator; +import org.opendatadiscovery.oddplatform.service.ingestion.util.DateTimeUtil; import org.opendatadiscovery.oddplatform.utils.JSONSerDeUtils; import org.opendatadiscovery.oddplatform.utils.Pair; import org.springframework.stereotype.Component; @@ -100,8 +103,8 @@ public DataEntityIngestionDto createIngestionDto(final DataEntity dataEntity, fi .oddrn(dataEntity.getOddrn()) .externalDescription(dataEntity.getDescription()) .dataSourceId(dataSourceId) - .createdAt(dataEntity.getCreatedAt()) - .updatedAt(dataEntity.getUpdatedAt()) + .sourceCreatedAt(dataEntity.getCreatedAt()) + .sourceUpdatedAt(dataEntity.getUpdatedAt()) .entityClasses(entityClasses) .type(type) .specificAttributesJson(specificAttributesAsString(entityClasses, dataEntity)); @@ -143,8 +146,9 @@ public DataEntityIngestionDto createIngestionDto(final DataEntity dataEntity, fi @Override public DataEntityPojo dtoToPojo(final T dto) { - final OffsetDateTime createdAt = dto.getCreatedAt(); - final OffsetDateTime updatedAt = dto.getUpdatedAt(); + final OffsetDateTime createdAt = dto.getSourceCreatedAt(); + final OffsetDateTime updatedAt = dto.getSourceUpdatedAt(); + final LocalDateTime now = DateTimeUtil.generateNow(); final Integer[] entityClassesIds = dto.getEntityClasses() .stream() @@ -157,16 +161,36 @@ public DataEntityPojo dtoToPojo(final T dto) .setOddrn(dto.getOddrn()) .setDataSourceId(dto.getDataSourceId()) .setEntityClassIds(entityClassesIds) - .setCreatedAt(createdAt != null ? createdAt.toLocalDateTime() : null) - .setUpdatedAt(updatedAt != null ? updatedAt.toLocalDateTime() : null) + .setPlatformCreatedAt(now) + .setSourceCreatedAt(createdAt != null ? createdAt.toLocalDateTime() : null) + .setSourceUpdatedAt(updatedAt != null ? updatedAt.toLocalDateTime() : null) + .setLastIngestedAt(now) .setTypeId(dto.getType().getId()) .setHollow(false) .setSpecificAttributes(JSONB.jsonb(dto.getSpecificAttributesJson())) .setExcludeFromSearch(isExcludedFromSearch(dto)) - .setManuallyCreated(false); + .setManuallyCreated(false) + .setStatus(DataEntityStatusDto.UNASSIGNED.getId()) + .setStatusUpdatedAt(now); if (dto instanceof EnrichedDataEntityIngestionDto entityIngestionDto) { - pojo.setId(entityIngestionDto.getId()); + final DataEntityPojo previousVersionPojo = entityIngestionDto.getPreviousVersionPojo(); + if (previousVersionPojo != null) { + pojo.setId(previousVersionPojo.getId()); + pojo.setInternalName(previousVersionPojo.getInternalName()); + pojo.setInternalDescription(previousVersionPojo.getInternalDescription()); + pojo.setViewCount(previousVersionPojo.getViewCount()); + pojo.setPlatformCreatedAt(previousVersionPojo.getPlatformCreatedAt()); + final DataEntityStatusDto previousStatus = getStatus(previousVersionPojo.getStatus()); + if (previousStatus == DataEntityStatusDto.DELETED) { + pojo.setStatus(DataEntityStatusDto.UNASSIGNED.getId()); + pojo.setStatusUpdatedAt(now); + } else { + pojo.setStatus(previousStatus.getId()); + pojo.setStatusUpdatedAt(previousVersionPojo.getStatusUpdatedAt()); + pojo.setStatusSwitchTime(previousVersionPojo.getStatusSwitchTime()); + } + } } return pojo; @@ -174,7 +198,7 @@ public DataEntityPojo dtoToPojo(final T dto) @Override public List dtoToPojo(final Collection dtos) { - return dtos.stream().map(this::dtoToPojo).collect(Collectors.toList()); + return dtos.stream().map(this::dtoToPojo).toList(); } @Override @@ -384,4 +408,9 @@ private boolean validateStructure(final List fieldList) { return true; } + + private DataEntityStatusDto getStatus(final Short statusId) { + return DataEntityStatusDto.findById(statusId) + .orElseThrow(() -> new IllegalArgumentException("Unknown DE status %s".formatted(statusId))); + } } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/notification/translator/AlertNotificationMessageTranslator.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/notification/translator/AlertNotificationMessageTranslator.java index 6c2a592cd..9b042714c 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/notification/translator/AlertNotificationMessageTranslator.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/notification/translator/AlertNotificationMessageTranslator.java @@ -154,14 +154,14 @@ private List fetchDownstream(final String rootDataEntityOddrn .select(LINEAGE.fields()) .select(startDepth) .from(LINEAGE) - .where(LINEAGE.PARENT_ODDRN.eq(rootDataEntityOddrn)) + .where(LINEAGE.PARENT_ODDRN.eq(rootDataEntityOddrn).and(LINEAGE.IS_DELETED.isFalse())) .unionAll( dslContext .select(LINEAGE.fields()) .select(depthField.add(1)) .from(LINEAGE) .join(cteName).on(LINEAGE.PARENT_ODDRN.eq(childOddrnField)) - .where(depthField.lessThan(downstreamEntitiesDepth + 1)) + .where(depthField.lessThan(downstreamEntitiesDepth + 1).and(LINEAGE.IS_DELETED.isFalse())) )); final Set downstreamEntitiesOddrns = dslContext.withRecursive(cte) diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/AlertHaltConfigRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/AlertHaltConfigRepositoryImpl.java index a08a86b53..b63be4def 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/AlertHaltConfigRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/AlertHaltConfigRepositoryImpl.java @@ -7,6 +7,7 @@ import org.jooq.Record; import org.jooq.SelectConditionStep; import org.jooq.impl.DSL; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; import org.opendatadiscovery.oddplatform.model.tables.pojos.AlertHaltConfigPojo; import org.opendatadiscovery.oddplatform.model.tables.records.AlertHaltConfigRecord; import org.opendatadiscovery.oddplatform.repository.util.JooqReactiveOperations; @@ -37,7 +38,7 @@ public Mono> getByOddrns(final Collection> listByOwner(final int page, final int size, final lo .from(ALERT) .join(DATA_ENTITY).on(DATA_ENTITY.ODDRN.eq(ALERT.DATA_ENTITY_ODDRN)) .join(OWNERSHIP).on(OWNERSHIP.DATA_ENTITY_ID.eq(DATA_ENTITY.ID)) - .where(ALERT.STATUS.eq(AlertStatusEnum.OPEN.getCode())).and(OWNERSHIP.OWNER_ID.eq(ownerId)); + .where(ALERT.STATUS.eq(AlertStatusEnum.OPEN.getCode())).and(OWNERSHIP.OWNER_ID.eq(ownerId) + .and(DATA_ENTITY.STATUS.ne(DataEntityStatusDto.DELETED.getId()))); final Pair, String> query = createAlertJoinQuery(baseQuery, (page - 1) * size, size); @@ -242,7 +244,7 @@ public Mono> getObjectsOddrnsByOwner(final long ownerId) { .select(DATA_ENTITY.ODDRN) .from(DATA_ENTITY) .leftJoin(OWNERSHIP).on(DATA_ENTITY.ID.eq(OWNERSHIP.DATA_ENTITY_ID)) - .where(OWNERSHIP.OWNER_ID.eq(ownerId).and(DATA_ENTITY.DELETED_AT.isNull())); + .where(OWNERSHIP.OWNER_ID.eq(ownerId).and(DATA_ENTITY.STATUS.ne(DataEntityStatusDto.DELETED.getId()))); return jooqReactiveOperations .flux(query) .map(r -> r.into(String.class)) @@ -431,13 +433,14 @@ private CommonTableExpression> getChildOddrnsLinageByOwnOddrnsCt .join(cteName) .on(LINEAGE.CHILD_ODDRN.eq( field(cteName.append(LINEAGE.PARENT_ODDRN.getUnqualifiedName()), String.class))) - .and(LINEAGE.PARENT_ODDRN.notEqual(DSL.all(parentOddrnArrayField))); + .and(LINEAGE.PARENT_ODDRN.notEqual(DSL.all(parentOddrnArrayField))) + .and(LINEAGE.IS_DELETED.isFalse()); final CommonTableExpression cte = cteName.as(DSL .select(LINEAGE.asterisk()) .select(parentOddrnArrayField) .from(LINEAGE) - .where(LINEAGE.CHILD_ODDRN.in(ownOddrns)) + .where(LINEAGE.CHILD_ODDRN.in(ownOddrns).and(LINEAGE.IS_DELETED.isFalse())) .unionAll(selectLinage)); return name("t2") diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityFilledRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityFilledRepositoryImpl.java index efdb0000d..503e02f5a 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityFilledRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityFilledRepositoryImpl.java @@ -6,6 +6,7 @@ import org.jooq.Field; import org.jooq.impl.DSL; import org.opendatadiscovery.oddplatform.dto.DataEntityFilledField; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityFilledPojo; import org.opendatadiscovery.oddplatform.repository.util.JooqReactiveOperations; import org.springframework.stereotype.Repository; @@ -24,7 +25,11 @@ public class ReactiveDataEntityFilledRepositoryImpl implements ReactiveDataEntit @Override public Mono getFilledDataEntitiesCount() { - return jooqReactiveOperations.mono(DSL.selectCount().from(DATA_ENTITY_FILLED)) + final var query = DSL.selectCount() + .from(DATA_ENTITY_FILLED) + .join(DATA_ENTITY).on(DATA_ENTITY_FILLED.DATA_ENTITY_ID.eq(DATA_ENTITY.ID)) + .where(DATA_ENTITY.STATUS.ne(DataEntityStatusDto.DELETED.getId())); + return jooqReactiveOperations.mono(query) .map(r -> r.into(Long.class)); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityRepository.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityRepository.java index 83609fb30..d5b5f9a35 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityRepository.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityRepository.java @@ -19,11 +19,11 @@ public interface ReactiveDataEntityRepository extends ReactiveCRUDRepository updateDEG(final DataEntityPojo dataEntityPojo); - Mono exists(final long dataEntityId); + Mono existsIncludingSoftDeleted(final long dataEntityId); - Mono existsByDataSourceId(final long dataSourceId); + Mono existsNonDeletedByDataSourceId(final long dataSourceId); - Mono existsByNamespaceId(final long namespaceId); + Mono existsNonDeletedByNamespaceId(final long namespaceId); Mono incrementViewCount(final long id); @@ -33,13 +33,19 @@ public interface ReactiveDataEntityRepository extends ReactiveCRUDRepository getDetails(final long id); - default Flux listAllByOddrns(final Collection oddrns, boolean includeHollow) { - return listAllByOddrns(oddrns, includeHollow, null, null); + default Flux listByOddrns(final Collection oddrns, + boolean includeHollow, + boolean includeDeleted) { + return listByOddrns(oddrns, includeHollow, includeDeleted, null, null); } - Flux listAllByOddrns(final Collection oddrns, - boolean includeHollow, - final Integer page, final Integer size); + Flux listByOddrns(final Collection oddrns, + boolean includeHollow, + boolean includeDeleted, + final Integer page, + final Integer size); + + Flux getPojosForStatusSwitch(); Mono getDataEntityWithDataSourceAndNamespace(final long dataEntityId); @@ -55,7 +61,7 @@ Mono> getDEGExperimentRuns(final Long dataEntityGr final Integer page, final Integer size); - Mono> getChildrenCount(final Collection groupOddrns); + Mono> getExperimentRunsCount(final Collection groupOddrns); Mono createHollow(final Collection hollowOddrns); diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityRepositoryImpl.java index 40c38a9bf..300443f66 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityRepositoryImpl.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -26,10 +27,12 @@ import org.opendatadiscovery.oddplatform.dto.DataEntityDimensionsDto; import org.opendatadiscovery.oddplatform.dto.DataEntityDomainInfoDto; import org.opendatadiscovery.oddplatform.dto.DataEntityDto; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; import org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto; import org.opendatadiscovery.oddplatform.dto.FacetStateDto; import org.opendatadiscovery.oddplatform.dto.FacetType; import org.opendatadiscovery.oddplatform.dto.OwnershipDto; +import org.opendatadiscovery.oddplatform.dto.SearchFilterDto; import org.opendatadiscovery.oddplatform.dto.alert.AlertStatusEnum; import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo; import org.opendatadiscovery.oddplatform.model.tables.pojos.DataSourcePojo; @@ -96,15 +99,38 @@ public ReactiveDataEntityRepositoryImpl(final JooqReactiveOperations jooqReactiv final JooqFTSHelper jooqFTSHelper, final DataEntityDtoMapper dataEntityDtoMapper) { super(jooqReactiveOperations, jooqQueryHelper, DATA_ENTITY, DataEntityPojo.class, DATA_ENTITY.EXTERNAL_NAME, - DATA_ENTITY.ID, DATA_ENTITY.CREATED_AT, DATA_ENTITY.UPDATED_AT, DATA_ENTITY.DELETED_AT); + DATA_ENTITY.ID, DATA_ENTITY.PLATFORM_CREATED_AT, null, null); this.jooqFTSHelper = jooqFTSHelper; this.jooqRecordHelper = jooqRecordHelper; this.dataEntityDtoMapper = dataEntityDtoMapper; } + @Override + protected Map, Object> getDeleteChangedFields() { + final Map, Object> updatedFieldsMap = new HashMap<>(); + updatedFieldsMap.put(DATA_ENTITY.STATUS, DataEntityStatusDto.DELETED.getId()); + updatedFieldsMap.put(DATA_ENTITY.STATUS_UPDATED_AT, DateTimeUtil.generateNow()); + updatedFieldsMap.put(DATA_ENTITY.STATUS_SWITCH_TIME, null); + return updatedFieldsMap; + } + + @Override + protected List addSoftDeleteFilter(final List conditions) { + final List conditionsList = new ArrayList<>(conditions); + conditionsList.add(DATA_ENTITY.STATUS.ne(DataEntityStatusDto.DELETED.getId())); + return conditionsList; + } + + @Override + public Mono get(final long id) { + return jooqReactiveOperations.mono(DSL.selectFrom(DATA_ENTITY).where(DATA_ENTITY.ID.eq(id))) + .map(this::recordToPojo); + } + @Override public Flux get(final List ids) { - return jooqReactiveOperations.flux(DSL.selectFrom(DATA_ENTITY).where(idCondition(ids))).map(this::recordToPojo); + return jooqReactiveOperations.flux(DSL.selectFrom(DATA_ENTITY).where(DATA_ENTITY.ID.in(ids))) + .map(this::recordToPojo); } @Override @@ -113,7 +139,6 @@ public Mono updateDEG(final DataEntityPojo dataEntityPojo) { .set(DATA_ENTITY.TYPE_ID, dataEntityPojo.getTypeId()) .set(DATA_ENTITY.NAMESPACE_ID, dataEntityPojo.getNamespaceId()) .set(DATA_ENTITY.INTERNAL_NAME, dataEntityPojo.getInternalName()) - .set(DATA_ENTITY.UPDATED_AT, dataEntityPojo.getUpdatedAt()) .where(DATA_ENTITY.ID.eq(dataEntityPojo.getId())) .returning(); return jooqReactiveOperations.mono(query) @@ -121,15 +146,15 @@ public Mono updateDEG(final DataEntityPojo dataEntityPojo) { } @Override - public Mono exists(final long dataEntityId) { + public Mono existsIncludingSoftDeleted(final long dataEntityId) { final Select> query = jooqQueryHelper.selectExists( - DSL.selectFrom(DATA_ENTITY).where(addSoftDeleteFilter(DATA_ENTITY.ID.eq(dataEntityId)))); + DSL.selectFrom(DATA_ENTITY).where(DATA_ENTITY.ID.eq(dataEntityId))); return jooqReactiveOperations.mono(query).map(Record1::component1).defaultIfEmpty(false); } @Override - public Mono existsByDataSourceId(final long dataSourceId) { + public Mono existsNonDeletedByDataSourceId(final long dataSourceId) { final Select> query = jooqQueryHelper.selectExists( DSL.selectFrom(DATA_ENTITY).where(addSoftDeleteFilter(DATA_ENTITY.DATA_SOURCE_ID.eq(dataSourceId)))); @@ -137,7 +162,7 @@ public Mono existsByDataSourceId(final long dataSourceId) { } @Override - public Mono existsByNamespaceId(final long namespaceId) { + public Mono existsNonDeletedByNamespaceId(final long namespaceId) { final Select> query = jooqQueryHelper.selectExists( DSL.selectFrom(DATA_ENTITY).where(addSoftDeleteFilter(DATA_ENTITY.NAMESPACE_ID.eq(namespaceId)))); @@ -157,6 +182,7 @@ public Mono incrementViewCount(final long id) { public Mono getDimensions(final long id) { final DataEntityCTEQueryConfig cteConfig = DataEntityCTEQueryConfig.builder() .conditions(List.of(DATA_ENTITY.ID.eq(id))) + .includeDeleted(true) .build(); final var query = baseDimensionsSelect(cteConfig); return jooqReactiveOperations.mono(query) @@ -178,6 +204,7 @@ public Mono> getDimensions(final Collection getDetails(final long id) { final DataEntityCTEQueryConfig cteConfig = DataEntityCTEQueryConfig.builder() .conditions(List.of(DATA_ENTITY.ID.eq(id))) + .includeDeleted(true) .build(); final var query = baseDimensionsSelect(cteConfig); return jooqReactiveOperations.mono(query) @@ -185,14 +212,21 @@ public Mono getDetails(final long id) { } @Override - public Flux listAllByOddrns(final Collection oddrns, - final boolean includeHollow, - final Integer page, - final Integer size) { + public Flux listByOddrns(final Collection oddrns, + final boolean includeHollow, + final boolean includeDeleted, + final Integer page, + final Integer size) { if (CollectionUtils.isEmpty(oddrns)) { return Flux.just(); } - final List conditions = new ArrayList<>(addSoftDeleteFilter(DATA_ENTITY.ODDRN.in(oddrns))); + final List conditions; + if (!includeDeleted) { + conditions = new ArrayList<>(addSoftDeleteFilter(DATA_ENTITY.ODDRN.in(oddrns))); + } else { + conditions = new ArrayList<>(); + conditions.add(DATA_ENTITY.ODDRN.in(oddrns)); + } if (!includeHollow) { conditions.add(DATA_ENTITY.HOLLOW.eq(false)); } @@ -205,6 +239,15 @@ public Flux listAllByOddrns(final Collection oddrns, return jooqReactiveOperations.flux(query).map(r -> r.into(DataEntityPojo.class)); } + @Override + public Flux getPojosForStatusSwitch() { + final var query = DSL.selectFrom(DATA_ENTITY) + .where(DATA_ENTITY.STATUS_SWITCH_TIME.isNotNull() + .and(DATA_ENTITY.STATUS_SWITCH_TIME.lessOrEqual(DateTimeUtil.generateNow()))); + return jooqReactiveOperations.flux(query) + .map(r -> r.into(DataEntityPojo.class)); + } + @Override public Mono getDataEntityWithDataSourceAndNamespace(final long dataEntityId) { final List conditions = addSoftDeleteFilter(DATA_ENTITY.ID.eq(dataEntityId)); @@ -263,7 +306,7 @@ public Mono> getDEGEntities(final String groupOddrn) { final SelectConditionStep query = DSL.select(DATA_ENTITY.fields()) .from(GROUP_ENTITY_RELATIONS) .leftJoin(DATA_ENTITY).on(DATA_ENTITY.ODDRN.eq(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN)) - .where(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.eq(groupOddrn)); + .where(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.eq(groupOddrn).and(GROUP_ENTITY_RELATIONS.IS_DELETED.isFalse())); return jooqReactiveOperations.flux(query) .map(r -> r.into(DataEntityPojo.class)) .collectList(); @@ -276,7 +319,7 @@ public Mono>> getDEGEntities(final Collection r.get(GROUP_ENTITY_RELATIONS.GROUP_ODDRN), @@ -318,7 +361,9 @@ public Mono> getDEGExperimentRuns(final Long dataE .from(DATA_ENTITY) .where(DATA_ENTITY.ID.eq(dataEntityGroupId)); final List conditions = List.of( - GROUP_PARENT_GROUP_RELATIONS.PARENT_GROUP_ODDRN.eq(dataEntityGroupOddrn)); + GROUP_PARENT_GROUP_RELATIONS.PARENT_GROUP_ODDRN.eq(dataEntityGroupOddrn) + .and(GROUP_PARENT_GROUP_RELATIONS.IS_DELETED.isFalse()) + ); final var query = DSL.with(deCteName) .asMaterialized(dataEntitySelect) @@ -337,12 +382,14 @@ public Mono> getDEGExperimentRuns(final Long dataE } @Override - public Mono> getChildrenCount(final Collection groupOddrns) { + public Mono> getExperimentRunsCount(final Collection groupOddrns) { final Field childrenCountField = field("children_count", Long.class); final var query = DSL.select(GROUP_PARENT_GROUP_RELATIONS.PARENT_GROUP_ODDRN) .select(count(GROUP_PARENT_GROUP_RELATIONS.GROUP_ODDRN).cast(Long.class).as(childrenCountField)) .from(GROUP_PARENT_GROUP_RELATIONS) - .where(GROUP_PARENT_GROUP_RELATIONS.PARENT_GROUP_ODDRN.in(groupOddrns)) + .join(DATA_ENTITY).on(GROUP_PARENT_GROUP_RELATIONS.GROUP_ODDRN.eq(DATA_ENTITY.ODDRN)) + .where(GROUP_PARENT_GROUP_RELATIONS.PARENT_GROUP_ODDRN.in(groupOddrns) + .and(GROUP_PARENT_GROUP_RELATIONS.IS_DELETED.isFalse())) .groupBy(GROUP_PARENT_GROUP_RELATIONS.PARENT_GROUP_ODDRN); return jooqReactiveOperations.flux(query).collectMap( r -> r.get(GROUP_PARENT_GROUP_RELATIONS.PARENT_GROUP_ODDRN), @@ -360,7 +407,6 @@ public Mono setInternalName(final long dataEntityId, final Strin final String newBusinessName = StringUtils.isEmpty(name) ? null : name; final var query = DSL.update(DATA_ENTITY) .set(DATA_ENTITY.INTERNAL_NAME, newBusinessName) - .set(DATA_ENTITY.UPDATED_AT, DateTimeUtil.generateNow()) .where(DATA_ENTITY.ID.eq(dataEntityId)) .returning(); return jooqReactiveOperations.mono(query) @@ -372,7 +418,6 @@ public Mono setInternalDescription(final long dataEntityId, fina final String newDescription = StringUtils.isEmpty(description) ? null : description; final var query = DSL.update(DATA_ENTITY) .set(DATA_ENTITY.INTERNAL_DESCRIPTION, newDescription) - .set(DATA_ENTITY.UPDATED_AT, DateTimeUtil.generateNow()) .where(DATA_ENTITY.ID.eq(dataEntityId)) .returning(); return jooqReactiveOperations.mono(query) @@ -383,8 +428,10 @@ public Mono setInternalDescription(final long dataEntityId, fina public Mono countByState(final FacetStateDto state, final OwnerPojo owner) { final List conditions = new ArrayList<>(jooqFTSHelper .facetStateConditions(state, DATA_ENTITY_CONDITIONS, List.of(FacetType.ENTITY_CLASSES))); + if (!deletedEntitiesAreRequested(state.getState())) { + conditions.add(DATA_ENTITY.STATUS.ne(DataEntityStatusDto.DELETED.getId())); + } conditions.add(DATA_ENTITY.HOLLOW.isFalse()); - conditions.add(DATA_ENTITY.DELETED_AT.isNull()); conditions.add(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isNull().or(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isFalse())); if (StringUtils.isNotEmpty(state.getQuery())) { conditions.add(jooqFTSHelper.ftsCondition(SEARCH_ENTRYPOINT.SEARCH_VECTOR, state.getQuery())); @@ -419,7 +466,7 @@ public Flux getQuerySuggestions(final String query, final List conditions = new ArrayList<>(); conditions.add(jooqFTSHelper.ftsCondition(SEARCH_ENTRYPOINT.SEARCH_VECTOR, query)); conditions.add(DATA_ENTITY.HOLLOW.isFalse()); - conditions.add(DATA_ENTITY.DELETED_AT.isNull()); + conditions.add(DATA_ENTITY.STATUS.ne(DataEntityStatusDto.DELETED.getId())); if (entityClassId != null) { conditions.add(DATA_ENTITY.ENTITY_CLASS_IDS.contains(new Integer[] {entityClassId})); } @@ -597,6 +644,9 @@ public Mono> findByState(final FacetStateDto state final Pair, List> conditionsPair = jooqFTSHelper.resultFacetStateConditions(state); final var builder = DataEntityCTEQueryConfig.builder() .conditions(conditionsPair.getLeft()); + if (deletedEntitiesAreRequested(state.getState())) { + builder.includeDeleted(true); + } if (StringUtils.isNotEmpty(state.getQuery())) { builder.fts(new DataEntityCTEQueryConfig.Fts(state.getQuery())); } @@ -621,6 +671,7 @@ public Mono> findByState(final FacetStateDto state jsonArrayAgg(field(OWNERSHIP.asterisk().toString())).as(AGG_OWNERSHIP_FIELD), hasAlerts(deCte)); + // DATA_ENTITY_TO_TERM and GROUP_ENTITY_RELATIONS are joint, because they are used in facet filters final Table fromTable = DSL.table(deCteName) .leftJoin(DATA_SOURCE) .on(DATA_SOURCE.ID.eq(jooqQueryHelper.getField(deCte, DATA_ENTITY.DATA_SOURCE_ID))) @@ -661,11 +712,12 @@ public Mono>> getParentDEGs(final Collection selectTable = cteSelect.asTable(cteName); final Field deOddrnField = @@ -760,6 +812,7 @@ public Flux getDataEntityDomainsInfo() { final String entitiesCountAlias = "entities_count"; final List conditions = getDataEntityDefaultConditions(); conditions.add(DATA_ENTITY.TYPE_ID.eq(DataEntityTypeDto.DOMAIN.getId())); + conditions.add(GROUP_ENTITY_RELATIONS.IS_DELETED.isFalse()); final var query = DSL.select(DATA_ENTITY.fields()) .select(coalesce(countDistinct(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN), 0L).as(entitiesCountAlias)) .from(DATA_ENTITY) @@ -774,21 +827,6 @@ public Flux getDataEntityDomainsInfo() { }); } - @Override - protected List> getNonUpdatableFields() { - final List> dataEntityNonUpdatableFields = List.of( - DATA_ENTITY.INTERNAL_NAME, - DATA_ENTITY.INTERNAL_DESCRIPTION, - DATA_ENTITY.VIEW_COUNT - ); - - // ad hoc until https://github.com/opendatadiscovery/odd-platform/issues/628 is closed - return ListUtils.union(dataEntityNonUpdatableFields, super.getNonUpdatableFields()) - .stream() - .filter(f -> !f.equals(DATA_ENTITY.CREATED_AT)) - .toList(); - } - private Select baseDataEntityWithDatasourceAndNamespaceSelect(final List conditions) { return DSL.select() .from(DATA_ENTITY) @@ -844,7 +882,12 @@ private Select baseDimensionsSelect(final DataEntityCTEQueryConfig cteCo private Select cteDataEntitySelect(final DataEntityCTEQueryConfig cteConfig) { final List> selectFields = new ArrayList<>(Arrays.stream(DATA_ENTITY.fields()).toList()); final Table fromTable; - final List conditions = addSoftDeleteFilter(ListUtils.emptyIfNull(cteConfig.getConditions())); + final List conditions = new ArrayList<>(); + if (cteConfig.isIncludeDeleted()) { + conditions.addAll(ListUtils.defaultIfNull(cteConfig.getConditions(), new ArrayList<>())); + } else { + conditions.addAll(addSoftDeleteFilter(ListUtils.emptyIfNull(cteConfig.getConditions()))); + } conditions.add(DATA_ENTITY.HOLLOW.isFalse()); if (cteConfig.getFts() != null) { final Field rankField = jooqFTSHelper @@ -900,8 +943,13 @@ private List> getOrderFields(final DataEntityCTEQueryConfig cteCon private List getDataEntityDefaultConditions() { final List conditions = new ArrayList<>(); conditions.add(DATA_ENTITY.HOLLOW.isFalse()); - conditions.add(DATA_ENTITY.DELETED_AT.isNull()); + conditions.add(DATA_ENTITY.STATUS.ne(DataEntityStatusDto.DELETED.getId())); conditions.add(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isNull().or(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isFalse())); return conditions; } + + private boolean deletedEntitiesAreRequested(final Map> facetStateMap) { + return facetStateMap.getOrDefault(FacetType.STATUSES, List.of()).stream() + .anyMatch(f -> f.getEntityId() == DataEntityStatusDto.DELETED.getId()); + } } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityTaskRunRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityTaskRunRepositoryImpl.java index f3c4bcb3b..9b5bad4ac 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityTaskRunRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityTaskRunRepositoryImpl.java @@ -163,7 +163,6 @@ public Mono> getDataEntityRuns(final long dataQualit final int size) { final List conditions = new ArrayList<>(); conditions.add(DATA_ENTITY.ID.eq(dataQualityTestId)); - conditions.add(DATA_ENTITY.HOLLOW.isFalse()); if (status != null) { conditions.add(DATA_ENTITY_TASK_RUN.STATUS.eq(status.name())); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataQualityRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataQualityRepositoryImpl.java index d615250f5..2f1dd22cc 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataQualityRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataQualityRepositoryImpl.java @@ -11,6 +11,7 @@ import org.jooq.SelectHavingStep; import org.jooq.impl.DSL; import org.opendatadiscovery.oddplatform.api.contract.model.DataQualityTestSeverity; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; import org.opendatadiscovery.oddplatform.dto.DatasetTestReportDto; import org.opendatadiscovery.oddplatform.dto.TestStatusWithSeverityDto; import org.opendatadiscovery.oddplatform.ingestion.contract.model.QualityRunStatus; @@ -37,21 +38,31 @@ @Repository @RequiredArgsConstructor public class ReactiveDataQualityRepositoryImpl implements ReactiveDataQualityRepository { + private static final String DATASET = "dataset"; + private static final String DATA_QA_TEST = "data_quality_test"; private final JooqReactiveOperations jooqReactiveOperations; @Override public Flux getDataQualityTestOddrnsForDataset(final long datasetId) { + final DataEntity dataset = DATA_ENTITY.as(DATASET); + final DataEntity dataQualityTest = DATA_ENTITY.as(DATA_QA_TEST); + final SelectConditionStep> query = DSL .select(DATA_QUALITY_TEST_RELATIONS.DATA_QUALITY_TEST_ODDRN) .from(DATA_QUALITY_TEST_RELATIONS) - .join(DATA_ENTITY).on(DATA_ENTITY.ODDRN.eq(DATA_QUALITY_TEST_RELATIONS.DATASET_ODDRN)) - .where(DATA_ENTITY.ID.eq(datasetId)).and(DATA_ENTITY.HOLLOW.isFalse()); + .join(dataset).on(dataset.ODDRN.eq(DATA_QUALITY_TEST_RELATIONS.DATASET_ODDRN)) + .join(dataQualityTest).on(dataQualityTest.ODDRN.eq(DATA_QUALITY_TEST_RELATIONS.DATA_QUALITY_TEST_ODDRN)) + .where(dataset.ID.eq(datasetId).and(dataset.HOLLOW.isFalse()) + .and(dataQualityTest.STATUS.ne(DataEntityStatusDto.DELETED.getId()))); return jooqReactiveOperations.flux(query).map(Record1::value1); } @Override public Mono getDatasetTestReport(final long datasetId) { + final DataEntity dataset = DATA_ENTITY.as(DATASET); + final DataEntity dataQualityTest = DATA_ENTITY.as(DATA_QA_TEST); + final SelectHavingStep> query = DSL .select( DATA_ENTITY_TASK_LAST_RUN.STATUS, @@ -60,9 +71,11 @@ public Mono getDatasetTestReport(final long datasetId) { .from(DATA_QUALITY_TEST_RELATIONS) .join(DATA_ENTITY_TASK_LAST_RUN) .on(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN.eq(DATA_QUALITY_TEST_RELATIONS.DATA_QUALITY_TEST_ODDRN)) - .join(DATA_ENTITY) - .on(DATA_ENTITY.ODDRN.eq(DATA_QUALITY_TEST_RELATIONS.DATASET_ODDRN)) - .where(DATA_ENTITY.ID.eq(datasetId)) + .join(dataset) + .on(dataset.ODDRN.eq(DATA_QUALITY_TEST_RELATIONS.DATASET_ODDRN)) + .join(dataQualityTest) + .on(dataQualityTest.ODDRN.eq(DATA_QUALITY_TEST_RELATIONS.DATA_QUALITY_TEST_ODDRN)) + .where(dataset.ID.eq(datasetId).and(dataQualityTest.STATUS.ne(DataEntityStatusDto.DELETED.getId()))) .groupBy(DATA_ENTITY_TASK_LAST_RUN.STATUS); return jooqReactiveOperations.flux(query) @@ -90,8 +103,8 @@ public Mono setDataQualityTestSeverity(final long d @Override public Flux getSLA(final long datasetId) { - final DataEntity dataset = DATA_ENTITY.as("dataset"); - final DataEntity dataQualityTest = DATA_ENTITY.as("data_quality_test"); + final DataEntity dataset = DATA_ENTITY.as(DATASET); + final DataEntity dataQualityTest = DATA_ENTITY.as(DATA_QA_TEST); // @formatter:off final SelectConditionStep> query = DSL @@ -106,7 +119,7 @@ public Flux getSLA(final long datasetId) { .leftJoin(DATA_QUALITY_TEST_SEVERITY) .on(DATA_QUALITY_TEST_SEVERITY.DATASET_ID.eq(dataset.ID)) .and(DATA_QUALITY_TEST_SEVERITY.DATA_QUALITY_TEST_ID.eq(dataQualityTest.ID)) - .where(dataset.ID.eq(datasetId)); + .where(dataset.ID.eq(datasetId).and(dataQualityTest.STATUS.ne(DataEntityStatusDto.DELETED.getId()))); // @formatter:on return jooqReactiveOperations.flux(query).map(this::mapLastRunDto); diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDatasetVersionRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDatasetVersionRepositoryImpl.java index 204e54eea..dbc147868 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDatasetVersionRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDatasetVersionRepositoryImpl.java @@ -96,7 +96,7 @@ public ReactiveDatasetVersionRepositoryImpl(final JooqReactiveOperations jooqRea public Mono getDatasetVersion(final long datasetVersionId) { final List> selectFields = Stream.of(DATASET_VERSION.fields(), DATASET_FIELD.fields()) .flatMap(Arrays::stream) - .collect(toList()); + .toList(); final SelectHavingStep selectHavingStep = DSL .select(selectFields) @@ -164,7 +164,7 @@ public Mono getLatestDatasetVersion(final long datasetId) { final List> selectFields = Stream.of(DATASET_VERSION.fields(), DATASET_FIELD.fields()) .flatMap(Arrays::stream) - .collect(toList()); + .toList(); final SelectHavingStep selectHavingStep = DSL .select(selectFields) diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupEntityRelationRepository.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupEntityRelationRepository.java index 5c1eb70a8..c6d27b448 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupEntityRelationRepository.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupEntityRelationRepository.java @@ -15,9 +15,11 @@ public interface ReactiveGroupEntityRelationRepository { Flux createRelationsReturning(final String groupOddrn, final List entityOddrns); - Mono deleteRelations(final List pojos); + Flux softDeleteRelationsForDeletedDataEntities(final List oddrns); + + Flux restoreRelationsForDataEntities(final List oddrns); - Flux deleteRelationsForDEG(final String groupOddrn); + Mono deleteRelations(final List pojos); Flux deleteRelationsExcept(final String groupOddrn, final List oddrnsToKeep); @@ -27,8 +29,6 @@ public interface ReactiveGroupEntityRelationRepository { Flux getDEGEntitiesOddrns(final long dataEntityGroupId); - Mono degHasEntities(final long dataEntityGroupId); - Flux getDEGItems(final Long dataEntityGroupId, final Integer page, final Integer size, final String query); diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupEntityRelationRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupEntityRelationRepositoryImpl.java index 3129b96e4..878ab2a4f 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupEntityRelationRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupEntityRelationRepositoryImpl.java @@ -12,8 +12,6 @@ import org.jooq.Field; import org.jooq.Name; import org.jooq.Record; -import org.jooq.Record1; -import org.jooq.Select; import org.jooq.SelectConditionStep; import org.jooq.impl.DSL; import org.opendatadiscovery.oddplatform.dto.DataEntityGroupItemDto; @@ -36,7 +34,6 @@ public class ReactiveGroupEntityRelationRepositoryImpl implements ReactiveGroupE private final JooqReactiveOperations jooqReactiveOperations; private final JooqQueryHelper jooqQueryHelper; - @Override public Flux deleteRelationsForDEG(final String groupOddrn) { final var query = DSL.deleteFrom(GROUP_ENTITY_RELATIONS) .where(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.eq(groupOddrn) @@ -92,6 +89,28 @@ public Flux createRelationsReturning(final String grou ).map(r -> r.into(GroupEntityRelationsPojo.class)); } + @Override + public Flux softDeleteRelationsForDeletedDataEntities(final List oddrns) { + final var query = DSL.update(GROUP_ENTITY_RELATIONS) + .set(GROUP_ENTITY_RELATIONS.IS_DELETED, true) + .where(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN.in(oddrns) + .or(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.in(oddrns))) + .returning(); + return jooqReactiveOperations.flux(query) + .map(r -> r.into(GroupEntityRelationsPojo.class)); + } + + @Override + public Flux restoreRelationsForDataEntities(final List oddrns) { + final var query = DSL.update(GROUP_ENTITY_RELATIONS) + .set(GROUP_ENTITY_RELATIONS.IS_DELETED, false) + .where(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN.in(oddrns) + .or(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.in(oddrns))) + .returning(); + return jooqReactiveOperations.flux(query) + .map(r -> r.into(GroupEntityRelationsPojo.class)); + } + @Override public Mono deleteRelations(final List pojos) { final List groupOddrns = pojos.stream().map(GroupEntityRelationsPojo::getGroupOddrn).toList(); @@ -109,7 +128,10 @@ public Flux getManuallyCreatedRelations(final String e final SelectConditionStep query = DSL.select() .from(GROUP_ENTITY_RELATIONS) .join(DATA_ENTITY).on(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.eq(DATA_ENTITY.ODDRN)) - .where(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN.eq(entityOddrn).and(DATA_ENTITY.MANUALLY_CREATED.isTrue())); + .where(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN.eq(entityOddrn) + .and(DATA_ENTITY.MANUALLY_CREATED.isTrue()) + .and(GROUP_ENTITY_RELATIONS.IS_DELETED.isFalse()) + ); return jooqReactiveOperations.flux(query) .map(r -> r.into(GroupEntityRelationsPojo.class)); } @@ -143,7 +165,8 @@ public Mono>> fetchGroupRelations(final Collection r.get(GROUP_ENTITY_RELATIONS.GROUP_ODDRN), @@ -163,12 +186,14 @@ public Flux getDEGEntitiesOddrns(final long dataEntityGroupId) { final var cte = cteName.as(DSL .select(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN) .from(GROUP_ENTITY_RELATIONS) - .where(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.eq(groupOddrn)) + .where(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.eq(groupOddrn) + .and(GROUP_ENTITY_RELATIONS.IS_DELETED.isFalse())) .unionAll( DSL .select(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN) .from(GROUP_ENTITY_RELATIONS) .join(cteName).on(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.eq(tDataEntityOddrn)) + .where(GROUP_ENTITY_RELATIONS.IS_DELETED.isFalse()) )); final var query = DSL.withRecursive(cte) @@ -178,19 +203,6 @@ public Flux getDEGEntitiesOddrns(final long dataEntityGroupId) { return jooqReactiveOperations.flux(query).map(r -> r.into(String.class)); } - @Override - public Mono degHasEntities(final long dataEntityGroupId) { - final var groupOddrn = DSL.select(DATA_ENTITY.ODDRN) - .from(DATA_ENTITY) - .where(DATA_ENTITY.ID.eq(dataEntityGroupId)); - - final Select> query = jooqQueryHelper.selectExists( - DSL.selectFrom(GROUP_ENTITY_RELATIONS) - .where(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.eq(groupOddrn)) - ); - return jooqReactiveOperations.mono(query).map(Record1::component1); - } - @Override public Flux getDEGItems(final Long dataEntityGroupId, final Integer page, final Integer size, final String query) { @@ -207,13 +219,15 @@ public Flux getDEGItems(final Long dataEntityGroupId, fi final var entitiesSelect = DSL.select(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN, DSL.inline(false)) .from(GROUP_ENTITY_RELATIONS) .join(DATA_ENTITY).on(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN.eq(DATA_ENTITY.ODDRN)) - .where(conditions).and(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.eq(groupOddrn)) + .where(conditions).and(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.eq(groupOddrn) + .and(GROUP_ENTITY_RELATIONS.IS_DELETED.isFalse())) .orderBy(DATA_ENTITY.ID.desc()); final var upperGroupsSelect = DSL.select(GROUP_ENTITY_RELATIONS.GROUP_ODDRN, DSL.inline(true)) .from(GROUP_ENTITY_RELATIONS) .join(DATA_ENTITY).on(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.eq(DATA_ENTITY.ODDRN)) - .where(conditions).and(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN.eq(groupOddrn)) + .where(conditions).and(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN.eq(groupOddrn) + .and(GROUP_ENTITY_RELATIONS.IS_DELETED.isFalse())) .orderBy(DATA_ENTITY.ID.desc()); final var resultQuery = entitiesSelect.unionAll(upperGroupsSelect) @@ -235,7 +249,8 @@ public Mono getDEGEntitiesCount(final Long dataEntityGroupId, final String final var result = DSL.selectCount() .from(GROUP_ENTITY_RELATIONS) .join(DATA_ENTITY).on(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN.eq(DATA_ENTITY.ODDRN)) - .where(conditions).and(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.eq(groupOddrn)); + .where(conditions).and(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.eq(groupOddrn) + .and(GROUP_ENTITY_RELATIONS.IS_DELETED.isFalse())); return jooqReactiveOperations.mono(result) .map(r -> r.value1().longValue()); @@ -252,7 +267,8 @@ public Mono getDEGUpperGroupsCount(final Long dataEntityGroupId, final Str final var result = DSL.selectCount() .from(GROUP_ENTITY_RELATIONS) .join(DATA_ENTITY).on(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.eq(DATA_ENTITY.ODDRN)) - .where(conditions).and(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN.eq(groupOddrn)); + .where(conditions).and(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN.eq(groupOddrn) + .and(GROUP_ENTITY_RELATIONS.IS_DELETED.isFalse())); return jooqReactiveOperations.mono(result) .map(r -> r.value1().longValue()); diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupParentGroupRelationRepository.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupParentGroupRelationRepository.java index 91b495120..887e30adc 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupParentGroupRelationRepository.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupParentGroupRelationRepository.java @@ -2,8 +2,13 @@ import java.util.List; import org.opendatadiscovery.oddplatform.model.tables.pojos.GroupParentGroupRelationsPojo; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; public interface ReactiveGroupParentGroupRelationRepository { Mono createRelations(final List pojos); + + Flux softDeleteRelationsForDeletedDataEntities(final List oddrns); + + Flux restoreRelationsForDataEntities(final List oddrns); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupParentGroupRelationRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupParentGroupRelationRepositoryImpl.java index bbb618975..5267a648c 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupParentGroupRelationRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveGroupParentGroupRelationRepositoryImpl.java @@ -6,6 +6,7 @@ import org.opendatadiscovery.oddplatform.model.tables.pojos.GroupParentGroupRelationsPojo; import org.opendatadiscovery.oddplatform.repository.util.JooqReactiveOperations; import org.springframework.stereotype.Repository; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import static org.opendatadiscovery.oddplatform.model.Tables.GROUP_PARENT_GROUP_RELATIONS; @@ -31,4 +32,26 @@ public Mono createRelations(final List pojo return jooqReactiveOperations.mono(step.onDuplicateKeyIgnore()); }); } + + @Override + public Flux softDeleteRelationsForDeletedDataEntities(final List oddrns) { + final var query = DSL.update(GROUP_PARENT_GROUP_RELATIONS) + .set(GROUP_PARENT_GROUP_RELATIONS.IS_DELETED, true) + .where(GROUP_PARENT_GROUP_RELATIONS.GROUP_ODDRN.in(oddrns) + .or(GROUP_PARENT_GROUP_RELATIONS.PARENT_GROUP_ODDRN.in(oddrns))) + .returning(); + return jooqReactiveOperations.flux(query) + .map(r -> r.into(GroupParentGroupRelationsPojo.class)); + } + + @Override + public Flux restoreRelationsForDataEntities(final List oddrns) { + final var query = DSL.update(GROUP_PARENT_GROUP_RELATIONS) + .set(GROUP_PARENT_GROUP_RELATIONS.IS_DELETED, false) + .where(GROUP_PARENT_GROUP_RELATIONS.GROUP_ODDRN.in(oddrns) + .or(GROUP_PARENT_GROUP_RELATIONS.PARENT_GROUP_ODDRN.in(oddrns))) + .returning(); + return jooqReactiveOperations.flux(query) + .map(r -> r.into(GroupParentGroupRelationsPojo.class)); + } } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveLineageRepository.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveLineageRepository.java index 6fbc61e84..598513fe0 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveLineageRepository.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveLineageRepository.java @@ -10,8 +10,7 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -public interface ReactiveLineageRepository extends ReactiveCRUDRepository { - +public interface ReactiveLineageRepository { Flux batchDeleteByEstablisherOddrn(Collection oddrns); Flux batchInsertLineages(final List pojos); @@ -30,4 +29,8 @@ Flux getLineageRelationsForDepthOne(final List rootIds, Mono> getChildrenCount(final Set oddrns); Mono> getParentCount(final Set oddrns); + + Flux softDeleteLineageRelations(final List dataEntityOddrns); + + Flux restoreLineageRelations(final List dataEntityOddrns); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveLineageRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveLineageRepositoryImpl.java index 30b19041e..52dd3a353 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveLineageRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveLineageRepositoryImpl.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import lombok.RequiredArgsConstructor; import org.apache.commons.collections4.CollectionUtils; import org.jooq.CommonTableExpression; import org.jooq.Condition; @@ -18,7 +19,6 @@ import org.opendatadiscovery.oddplatform.dto.lineage.LineageStreamKind; import org.opendatadiscovery.oddplatform.model.tables.pojos.LineagePojo; import org.opendatadiscovery.oddplatform.model.tables.records.LineageRecord; -import org.opendatadiscovery.oddplatform.repository.util.JooqQueryHelper; import org.opendatadiscovery.oddplatform.repository.util.JooqReactiveOperations; import org.opendatadiscovery.oddplatform.utils.Pair; import org.springframework.stereotype.Repository; @@ -35,12 +35,9 @@ import static org.opendatadiscovery.oddplatform.model.Tables.LINEAGE; @Repository -public class ReactiveLineageRepositoryImpl extends ReactiveAbstractCRUDRepository - implements ReactiveLineageRepository { - public ReactiveLineageRepositoryImpl(final JooqReactiveOperations jooqReactiveOperations, - final JooqQueryHelper jooqQueryHelper) { - super(jooqReactiveOperations, jooqQueryHelper, LINEAGE, LineagePojo.class); - } +@RequiredArgsConstructor +public class ReactiveLineageRepositoryImpl implements ReactiveLineageRepository { + private final JooqReactiveOperations jooqReactiveOperations; @Override public Flux batchDeleteByEstablisherOddrn(final Collection establishers) { @@ -65,7 +62,7 @@ public Mono> getTargetsCount(final Set oddrns) { final var query = DSL.select(LINEAGE.PARENT_ODDRN) .select(count(one()).cast(Long.class)) .from(LINEAGE) - .where(LINEAGE.PARENT_ODDRN.in(oddrns)) + .where(LINEAGE.PARENT_ODDRN.in(oddrns).and(LINEAGE.IS_DELETED.isFalse())) .groupBy(LINEAGE.PARENT_ODDRN); return jooqReactiveOperations.flux(query) .collectMap(r -> r.get(0, String.class), r -> r.get(1, Long.class)); @@ -76,7 +73,7 @@ public Mono> getChildrenCount(final Set oddrns) { final Field childrenCount = countDistinct(LINEAGE.CHILD_ODDRN).as("children_count"); final var query = DSL.select(LINEAGE.PARENT_ODDRN, childrenCount) .from(LINEAGE) - .where(LINEAGE.PARENT_ODDRN.in(oddrns)) + .where(LINEAGE.PARENT_ODDRN.in(oddrns).and(LINEAGE.IS_DELETED.isFalse())) .groupBy(LINEAGE.PARENT_ODDRN); return jooqReactiveOperations.flux(query).collectMap(Record2::value1, Record2::value2); } @@ -86,17 +83,38 @@ public Mono> getParentCount(final Set oddrns) { final Field parentsCount = countDistinct(LINEAGE.PARENT_ODDRN).as("parents_count"); final var query = DSL.select(LINEAGE.CHILD_ODDRN, parentsCount) .from(LINEAGE) - .where(LINEAGE.CHILD_ODDRN.in(oddrns)) + .where(LINEAGE.CHILD_ODDRN.in(oddrns).and(LINEAGE.IS_DELETED.isFalse())) .groupBy(LINEAGE.CHILD_ODDRN); return jooqReactiveOperations.flux(query).collectMap(Record2::value1, Record2::value2); } + @Override + public Flux softDeleteLineageRelations(final List dataEntityOddrns) { + final var query = DSL.update(LINEAGE) + .set(LINEAGE.IS_DELETED, true) + .where(LINEAGE.CHILD_ODDRN.in(dataEntityOddrns).or(LINEAGE.PARENT_ODDRN.in(dataEntityOddrns))) + .returning(); + return jooqReactiveOperations.flux(query) + .map(r -> r.into(LineagePojo.class)); + } + + @Override + public Flux restoreLineageRelations(final List dataEntityOddrns) { + final var query = DSL.update(LINEAGE) + .set(LINEAGE.IS_DELETED, false) + .where(LINEAGE.CHILD_ODDRN.in(dataEntityOddrns).or(LINEAGE.PARENT_ODDRN.in(dataEntityOddrns))) + .returning(); + return jooqReactiveOperations.flux(query) + .map(r -> r.into(LineagePojo.class)); + } + @Override public Flux getLineageRelations(final List oddrns) { final var query = DSL.selectDistinct(LINEAGE.PARENT_ODDRN, LINEAGE.CHILD_ODDRN) .from(LINEAGE) - .where(LINEAGE.PARENT_ODDRN.in(oddrns).and(LINEAGE.CHILD_ODDRN.in(oddrns)) - .or(LINEAGE.CHILD_ODDRN.in(oddrns).and(LINEAGE.PARENT_ODDRN.in(oddrns)))); + .where(LINEAGE.IS_DELETED.isFalse() + .and(LINEAGE.PARENT_ODDRN.in(oddrns).and(LINEAGE.CHILD_ODDRN.in(oddrns)) + .or(LINEAGE.CHILD_ODDRN.in(oddrns).and(LINEAGE.PARENT_ODDRN.in(oddrns))))); return jooqReactiveOperations.flux(query).map(r -> r.into(LineagePojo.class)); } @@ -124,7 +142,7 @@ public Flux getLineageRelationsForDepthOne(final List rootIds final var query = DSL.selectDistinct(LINEAGE.PARENT_ODDRN, LINEAGE.CHILD_ODDRN) .from(LINEAGE) .join(DATA_ENTITY).on(joinCondition) - .where(DATA_ENTITY.ID.in(rootIds)); + .where(DATA_ENTITY.ID.in(rootIds).and(LINEAGE.IS_DELETED.isFalse())); return jooqReactiveOperations.flux(query) .map(r -> r.into(LineagePojo.class)); } @@ -146,14 +164,14 @@ private CommonTableExpression lineageCte(final Collection oddrns .select(LINEAGE.asterisk()) .select(startDepth) .from(LINEAGE) - .where(conditions.getLeft().in(oddrns)) + .where(conditions.getLeft().in(oddrns).and(LINEAGE.IS_DELETED.isFalse())) .unionAll( DSL .select(LINEAGE.asterisk()) .select(tDepth.add(1)) .from(LINEAGE) .join(cteName).on(conditions.getLeft().eq(conditions.getRight())) - .where(tDepth.lessThan(lineageDepth.getDepth())) + .where(tDepth.lessThan(lineageDepth.getDepth()).and(LINEAGE.IS_DELETED.isFalse())) )); } } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveOwnershipRepository.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveOwnershipRepository.java index 0a11e1055..8035c2d84 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveOwnershipRepository.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveOwnershipRepository.java @@ -21,7 +21,5 @@ public interface ReactiveOwnershipRepository { Mono existsByOwner(final long ownerId); - Flux deleteByDataEntityId(final long dataEntityId); - Flux getOwnershipsByDataEntityId(final long dataEntityId); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveOwnershipRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveOwnershipRepositoryImpl.java index 168f2a5e7..30f1294e7 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveOwnershipRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveOwnershipRepositoryImpl.java @@ -126,15 +126,6 @@ public Mono existsByOwner(final long ownerId) { return jooqReactiveOperations.mono(query).map(Record1::component1); } - @Override - public Flux deleteByDataEntityId(final long dataEntityId) { - final var query = DSL.deleteFrom(OWNERSHIP) - .where(OWNERSHIP.DATA_ENTITY_ID.eq(dataEntityId)) - .returning(); - return jooqReactiveOperations.flux(query) - .map(r -> r.into(OwnershipPojo.class)); - } - @Override public Flux getOwnershipsByDataEntityId(final long dataEntityId) { final var query = DSL.select(OWNERSHIP.asterisk()) diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchEntrypointRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchEntrypointRepositoryImpl.java index 02fd204f4..1430441d5 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchEntrypointRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchEntrypointRepositoryImpl.java @@ -17,6 +17,7 @@ import org.jooq.UpdateConditionStep; import org.jooq.impl.DSL; import org.opendatadiscovery.oddplatform.annotation.ReactiveTransactional; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; import org.opendatadiscovery.oddplatform.model.Tables; import org.opendatadiscovery.oddplatform.model.tables.records.SearchEntrypointRecord; import org.opendatadiscovery.oddplatform.repository.util.FTSEntity; @@ -110,7 +111,7 @@ public Mono updateDataSourceVectorsForDataEntities(final List dat final SelectConditionStep vectorSelect = DSL .select(DATA_ENTITY.ID.as(dataEntityIdField)) .select(vectorFields) - .from(Tables.DATA_SOURCE) + .from(DATA_SOURCE) .join(DATA_ENTITY).on(DATA_ENTITY.DATA_SOURCE_ID.eq(Tables.DATA_SOURCE.ID)) .and(DATA_ENTITY.HOLLOW.isFalse()) .and(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isNull().or(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isFalse())) @@ -128,6 +129,7 @@ public Mono updateDataSourceVectorsForDataEntities(final List dat return jooqReactiveOperations.mono(insertQuery); } + @Override public Mono updateNamespaceVectorForDataEntity(final long dataEntityId) { return updateNamespaceVectorForDataEntities(singletonList(dataEntityId)); } @@ -171,10 +173,13 @@ public Mono updateChangedNamespaceVector(final long namespaceId) { .select(vectorFields) .from(NAMESPACE) .join(DATA_SOURCE).on(DATA_SOURCE.NAMESPACE_ID.eq(NAMESPACE.ID)) - .join(DATA_ENTITY).on(DATA_ENTITY.DATA_SOURCE_ID.eq(DATA_SOURCE.ID)).and(DATA_ENTITY.HOLLOW.isFalse()) - .and(DATA_ENTITY.DELETED_AT.isNull()) + .join(DATA_ENTITY).on(DATA_ENTITY.DATA_SOURCE_ID.eq(DATA_SOURCE.ID)) + .or(DATA_ENTITY.NAMESPACE_ID.eq(NAMESPACE.ID)) .where(NAMESPACE.ID.eq(namespaceId)) - .and(NAMESPACE.DELETED_AT.isNull()); + .and(NAMESPACE.DELETED_AT.isNull()) + .and(DATA_ENTITY.HOLLOW.isFalse()) + .and(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isNull().or(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isFalse())) + .and(DATA_ENTITY.STATUS.ne(DataEntityStatusDto.DELETED.getId())); final Insert insertQuery = jooqFTSHelper.buildVectorUpsert( vectorSelect, @@ -355,7 +360,7 @@ public Mono updateChangedOwnerVectors(final long ownerId) { .join(TITLE).on(TITLE.ID.eq(OWNERSHIP.TITLE_ID)) .join(DATA_ENTITY).on(DATA_ENTITY.ID.eq(OWNERSHIP.DATA_ENTITY_ID)) .and(DATA_ENTITY.HOLLOW.isFalse()) - .and(DATA_ENTITY.DELETED_AT.isNull()) + .and(DATA_ENTITY.STATUS.ne(DataEntityStatusDto.DELETED.getId())) .where(OWNER.ID.eq(ownerId)); final Insert ownerQuery = jooqFTSHelper.buildVectorUpsert( diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepository.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepository.java index 6769a55c7..e434e9552 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepository.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepository.java @@ -46,4 +46,9 @@ Mono> getGroupFacetForDataEntity(final String facetQue final int page, final int size, final FacetStateDto state); + + Mono> getStatusFacetForDataEntity(final String query, + final int page, + final int size, + final FacetStateDto state); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepositoryImpl.java index 0671de77f..1079f00d9 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepositoryImpl.java @@ -18,9 +18,11 @@ import org.jooq.Table; import org.jooq.impl.DSL; import org.opendatadiscovery.oddplatform.dto.DataEntityClassDto; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; import org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto; import org.opendatadiscovery.oddplatform.dto.FacetStateDto; import org.opendatadiscovery.oddplatform.dto.FacetType; +import org.opendatadiscovery.oddplatform.dto.SearchFilterDto; import org.opendatadiscovery.oddplatform.dto.SearchFilterId; import org.opendatadiscovery.oddplatform.model.tables.pojos.SearchFacetsPojo; import org.opendatadiscovery.oddplatform.model.tables.records.SearchFacetsRecord; @@ -153,7 +155,12 @@ public Mono> getOwnerFacetForTerms(final String facetQ @Override public Mono> getEntityClassFacetForDataEntity(final FacetStateDto state) { - final List conditions = getDataEntityDefaultConditions(); + final List conditions = new ArrayList<>(); + conditions.add(DATA_ENTITY.HOLLOW.isFalse()); + conditions.add(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isNull().or(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isFalse())); + if (!deletedEntitiesAreRequested(state.getState())) { + conditions.add(DATA_ENTITY.STATUS.ne(DataEntityStatusDto.DELETED.getId())); + } final String entityClassUnnestedField = "entity_class_id"; final String deCountField = "data_entity_count"; @@ -207,6 +214,11 @@ public Mono> getEntityClassFacetForDataEntity(final Fa conditions.add(DATA_ENTITY.TYPE_ID.in(typeIds)); } + final Set statusIds = state.getFacetEntitiesIds(FacetType.STATUSES); + if (!CollectionUtils.isEmpty(statusIds)) { + conditions.add(DATA_ENTITY.STATUS.in(statusIds)); + } + select .where(conditions) .groupBy(field(entityClassUnnestedField)); @@ -432,6 +444,88 @@ public Mono> getGroupFacetForDataEntity(final String f .collect(FACET_COLLECTOR); } + @Override + public Mono> getStatusFacetForDataEntity(final String query, + final int page, + final int size, + final FacetStateDto state) { + final List conditions = new ArrayList<>(); + conditions.add(DATA_ENTITY.HOLLOW.isFalse()); + conditions.add(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isNull().or(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isFalse())); + conditions.addAll(getQueryAndEntityClassConditions(state)); + var select = DSL + .select(DATA_ENTITY.STATUS, count(DATA_ENTITY.ID)) + .from(DATA_ENTITY); + + if (StringUtils.isNotEmpty(state.getQuery())) { + select.join(SEARCH_ENTRYPOINT).on(SEARCH_ENTRYPOINT.DATA_ENTITY_ID.eq(DATA_ENTITY.ID)); + conditions.add(jooqFTSHelper.ftsCondition(SEARCH_ENTRYPOINT.SEARCH_VECTOR, state.getQuery())); + } + + final Set dataSourceIds = state.getFacetEntitiesIds(FacetType.DATA_SOURCES); + if (!CollectionUtils.isEmpty(dataSourceIds)) { + conditions.add(DATA_ENTITY.DATA_SOURCE_ID.in(dataSourceIds)); + } + final Set ownerIds = state.getFacetEntitiesIds(FacetType.OWNERS); + if (!CollectionUtils.isEmpty(ownerIds)) { + select.join(OWNERSHIP).on(OWNERSHIP.DATA_ENTITY_ID.eq(DATA_ENTITY.ID)); + conditions.add(OWNERSHIP.OWNER_ID.in(ownerIds)); + } + final Set namespaceIds = state.getFacetEntitiesIds(FacetType.NAMESPACES); + if (!CollectionUtils.isEmpty(namespaceIds)) { + select.leftJoin(DATA_SOURCE) + .on(DATA_SOURCE.ID.eq(DATA_ENTITY.DATA_SOURCE_ID)) + .leftJoin(NAMESPACE).on(NAMESPACE.ID.eq(DATA_ENTITY.NAMESPACE_ID)) + .or(NAMESPACE.ID.eq(DATA_SOURCE.NAMESPACE_ID)); + conditions.add(NAMESPACE.ID.in(namespaceIds)); + } + final Set tagIds = state.getFacetEntitiesIds(FacetType.TAGS); + if (!CollectionUtils.isEmpty(tagIds)) { + select = select.join(TAG_TO_DATA_ENTITY) + .on(TAG_TO_DATA_ENTITY.DATA_ENTITY_ID.eq(DATA_ENTITY.ID)); + conditions.add(TAG_TO_DATA_ENTITY.TAG_ID.in(tagIds)); + } + + final Set groupIds = state.getFacetEntitiesIds(FacetType.GROUPS); + if (!CollectionUtils.isEmpty(groupIds)) { + select = select.join(GROUP_ENTITY_RELATIONS) + .on(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN.eq(DATA_ENTITY.ODDRN)); + + final var groupOddrns = DSL.select(DATA_ENTITY.ODDRN) + .from(DATA_ENTITY) + .where(DATA_ENTITY.ID.in(groupIds)); + conditions.add(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.in(groupOddrns)); + } + final List statusIds = statusIdsByName(query); + if (!statusIds.isEmpty()) { + conditions.add(DATA_ENTITY.STATUS.in(statusIds)); + } + + select + .where(conditions) + .groupBy(DATA_ENTITY.STATUS) + .orderBy(count(DATA_ENTITY.ID).desc()) + .limit(size) + .offset((page - 1) * size); + + final Flux> existingStatuses = jooqReactiveOperations.flux(select) + .map(r -> { + final DataEntityStatusDto status = DataEntityStatusDto.findById(r.component1()) + .orElseThrow(() -> new IllegalArgumentException( + String.format("There's no status with id %d", r.component1()))); + + return Pair.of(statusToSearchFilter(status), r.component2().longValue()); + }); + + final Flux> all = Flux.fromStream(Arrays.stream(DataEntityStatusDto.values())) + .map(s -> Pair.of(statusToSearchFilter(s), 0L)); + + return Flux.concat(existingStatuses, all) + .filter(s -> StringUtils.isEmpty(query) + || StringUtils.containsIgnoreCase(query, s.getLeft().getName())) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (t1, t2) -> t1 == 0 ? t2 : t1)); + } + private List typeIdsByName(final String name) { return Arrays.stream(DataEntityTypeDto.values()) .filter(s -> StringUtils.containsIgnoreCase(name, s.name())) @@ -439,6 +533,13 @@ private List typeIdsByName(final String name) { .toList(); } + private List statusIdsByName(final String name) { + return Arrays.stream(DataEntityStatusDto.values()) + .filter(s -> StringUtils.containsIgnoreCase(name, s.name())) + .map(DataEntityStatusDto::getId) + .toList(); + } + private SearchFilterId typeToSearchFilter(final DataEntityTypeDto type) { return SearchFilterId.builder() .entityId(type.getId()) @@ -453,10 +554,17 @@ private SearchFilterId entityClassToSearchFilter(final DataEntityClassDto entity .build(); } + private SearchFilterId statusToSearchFilter(final DataEntityStatusDto status) { + return SearchFilterId.builder() + .entityId(status.getId()) + .name(status.name()) + .build(); + } + private List getDataEntityDefaultConditions() { final List conditions = new ArrayList<>(); conditions.add(DATA_ENTITY.HOLLOW.isFalse()); - conditions.add(DATA_ENTITY.DELETED_AT.isNull()); + conditions.add(DATA_ENTITY.STATUS.ne(DataEntityStatusDto.DELETED.getId())); conditions.add(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isNull().or(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isFalse())); return conditions; } @@ -472,4 +580,9 @@ private List getQueryAndEntityClassConditions(final FacetStateDto sta } return conditions; } + + private boolean deletedEntitiesAreRequested(final Map> facetStateMap) { + return facetStateMap.getOrDefault(FacetType.STATUSES, List.of()).stream() + .anyMatch(f -> f.getEntityId() == DataEntityStatusDto.DELETED.getId()); + } } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveTagRepository.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveTagRepository.java index 4182f352b..0ca7d719c 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveTagRepository.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveTagRepository.java @@ -31,8 +31,6 @@ public interface ReactiveTagRepository extends ReactiveCRUDRepository { Flux deleteDataEntityRelations(final long tagId); - Flux deleteRelationsForDataEntity(final long dataEntityId); - Flux createTermRelations(final long termId, final Collection tagIds); Flux deleteTermRelations(final long termId, final Collection tagIds); diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveTagRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveTagRepositoryImpl.java index dc47ae3d0..ce57ec761 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveTagRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveTagRepositoryImpl.java @@ -201,15 +201,6 @@ public Flux deleteDataEntityRelations(final long tagId) { .map(r -> r.into(TagToDataEntityPojo.class)); } - @Override - public Flux deleteRelationsForDataEntity(final long dataEntityId) { - final var query = DSL.delete(TAG_TO_DATA_ENTITY) - .where(TAG_TO_DATA_ENTITY.DATA_ENTITY_ID.eq(dataEntityId)) - .returning(); - return jooqReactiveOperations.flux(query) - .map(r -> r.into(TagToDataEntityPojo.class)); - } - @Override public Flux createDataEntityRelations(final Collection relations) { if (CollectionUtils.isEmpty(relations)) { diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveTermRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveTermRepositoryImpl.java index 0bae11b3d..feccad459 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveTermRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveTermRepositoryImpl.java @@ -18,6 +18,7 @@ import org.jooq.SortOrder; import org.jooq.Table; import org.jooq.impl.DSL; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; import org.opendatadiscovery.oddplatform.dto.FacetStateDto; import org.opendatadiscovery.oddplatform.dto.FacetType; import org.opendatadiscovery.oddplatform.dto.term.LinkedTermDto; @@ -48,7 +49,11 @@ import static org.jooq.impl.DSL.exists; import static org.jooq.impl.DSL.field; import static org.jooq.impl.DSL.jsonArrayAgg; +import static org.opendatadiscovery.oddplatform.model.Tables.DATASET_FIELD; import static org.opendatadiscovery.oddplatform.model.Tables.DATASET_FIELD_TO_TERM; +import static org.opendatadiscovery.oddplatform.model.Tables.DATASET_STRUCTURE; +import static org.opendatadiscovery.oddplatform.model.Tables.DATASET_VERSION; +import static org.opendatadiscovery.oddplatform.model.Tables.DATA_ENTITY; import static org.opendatadiscovery.oddplatform.model.Tables.DATA_ENTITY_TO_TERM; import static org.opendatadiscovery.oddplatform.model.Tables.NAMESPACE; import static org.opendatadiscovery.oddplatform.model.Tables.OWNER; @@ -358,10 +363,21 @@ public Flux getDatasetFieldTerms(final long datasetFieldId) { public Mono hasDescriptionRelations(final long termId) { final Condition dataEntityDescriptionRelations = exists(DSL.selectOne() .from(DATA_ENTITY_TO_TERM) - .where(DATA_ENTITY_TO_TERM.TERM_ID.eq(termId).and(DATA_ENTITY_TO_TERM.IS_DESCRIPTION_LINK.isTrue()))); + .join(DATA_ENTITY).on(DATA_ENTITY_TO_TERM.DATA_ENTITY_ID.eq(DATA_ENTITY.ID)) + .where(DATA_ENTITY_TO_TERM.TERM_ID.eq(termId) + .and(DATA_ENTITY_TO_TERM.IS_DESCRIPTION_LINK.isTrue()) + .and(DATA_ENTITY.STATUS.ne(DataEntityStatusDto.DELETED.getId())) + )); final Condition datasetFieldDescriptionRelations = exists(DSL.selectOne() .from(DATASET_FIELD_TO_TERM) - .where(DATASET_FIELD_TO_TERM.TERM_ID.eq(termId).and(DATASET_FIELD_TO_TERM.IS_DESCRIPTION_LINK.isTrue()))); + .join(DATASET_FIELD).on(DATASET_FIELD_TO_TERM.DATASET_FIELD_ID.eq(DATASET_FIELD.ID)) + .join(DATASET_STRUCTURE).on(DATASET_FIELD.ID.eq(DATASET_STRUCTURE.DATASET_FIELD_ID)) + .join(DATASET_VERSION).on(DATASET_VERSION.ID.eq(DATASET_STRUCTURE.DATASET_VERSION_ID)) + .join(DATA_ENTITY).on(DATA_ENTITY.ODDRN.eq(DATASET_VERSION.DATASET_ODDRN)) + .where(DATASET_FIELD_TO_TERM.TERM_ID.eq(termId) + .and(DATASET_FIELD_TO_TERM.IS_DESCRIPTION_LINK.isTrue()) + .and(DATA_ENTITY.STATUS.ne(DataEntityStatusDto.DELETED.getId())) + )); final var query = DSL.select(dataEntityDescriptionRelations.or(datasetFieldDescriptionRelations)); return jooqReactiveOperations.mono(query).map(Record1::component1); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/TermRelationsRepository.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/TermRelationsRepository.java index ad8bd3ce2..fbb6bafb9 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/TermRelationsRepository.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/TermRelationsRepository.java @@ -11,8 +11,6 @@ public interface TermRelationsRepository { Flux createRelationsWithDataEntity(final List relations); - Flux deleteRelationsWithTerms(final long dataEntityId); - Flux deleteRelationsWithDataEntities(final long termId); Flux deleteRelationsWithDatasetFields(final long termId); diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/TermRelationsRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/TermRelationsRepositoryImpl.java index f4941d3c9..760667aff 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/TermRelationsRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/TermRelationsRepositoryImpl.java @@ -55,15 +55,6 @@ public Flux createRelationsWithDataEntity(final List r.into(DataEntityToTermPojo.class)); } - @Override - public Flux deleteRelationsWithTerms(final long dataEntityId) { - final var query = DSL.deleteFrom(DATA_ENTITY_TO_TERM) - .where(DATA_ENTITY_TO_TERM.DATA_ENTITY_ID.eq(dataEntityId)) - .returning(); - return jooqReactiveOperations.flux(query) - .map(r -> r.into(DataEntityToTermPojo.class)); - } - @Override public Flux deleteRelationsWithDataEntities(final long termId) { final var query = DSL.deleteFrom(DATA_ENTITY_TO_TERM) diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/DataEntityCTEQueryConfig.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/DataEntityCTEQueryConfig.java index a9e79c3c2..1227391e2 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/DataEntityCTEQueryConfig.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/DataEntityCTEQueryConfig.java @@ -27,6 +27,7 @@ public class DataEntityCTEQueryConfig { private LimitOffset limitOffset; private SortField orderBy; private Fts fts; + private boolean includeDeleted; public record LimitOffset(int limit, int offset) { } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/FTSConstants.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/FTSConstants.java index c3cb4e7b9..4826e8138 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/FTSConstants.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/FTSConstants.java @@ -70,7 +70,8 @@ public class FTSConstants { .from(DATA_ENTITY) .where(DATA_ENTITY.ID.in(extractFilterId(filters))); return GROUP_ENTITY_RELATIONS.GROUP_ODDRN.in(groupOddrns); - } + }, + FacetType.STATUSES, filters -> DATA_ENTITY.STATUS.in(extractFilterId(filters)) ); public static final Map, Condition>> TERM_CONDITIONS = Map.of( diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/JooqFTSHelper.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/JooqFTSHelper.java index fd37ded23..a6e1ac93d 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/JooqFTSHelper.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/JooqFTSHelper.java @@ -118,14 +118,15 @@ public List facetStateConditions( .filter(e -> !ignoredFacets.contains(e.getKey())) .map(e -> compileFacetCondition(e.getKey(), e.getValue(), facetTypeFunctionMap)) .filter(Objects::nonNull) - .collect(Collectors.toList()); + .toList(); } public Pair, List> resultFacetStateConditions(final FacetStateDto state) { final Predicate>> cteFilters = e -> e.getKey().equals(FacetType.DATA_SOURCES) || e.getKey().equals(FacetType.ENTITY_CLASSES) - || e.getKey().equals(FacetType.TYPES); + || e.getKey().equals(FacetType.TYPES) + || e.getKey().equals(FacetType.STATUSES); final List joinConditions = state.getState().entrySet().stream() .filter(not(cteFilters)) diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/AlertHaltConfigServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/AlertHaltConfigServiceImpl.java index c05f2ced3..665eac555 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/AlertHaltConfigServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/AlertHaltConfigServiceImpl.java @@ -24,7 +24,7 @@ public class AlertHaltConfigServiceImpl implements AlertHaltConfigService { @Override public Mono getAlertHaltConfig(final long dataEntityId) { - return dataEntityRepository.exists(dataEntityId) + return dataEntityRepository.existsIncludingSoftDeleted(dataEntityId) .filter(exists -> exists) .switchIfEmpty(Mono.error(() -> new NotFoundException("data entity", dataEntityId))) .then(alertHaltConfigRepository.get(dataEntityId)) diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/AlertServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/AlertServiceImpl.java index 19bc8d903..956b500fe 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/AlertServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/AlertServiceImpl.java @@ -325,7 +325,7 @@ private Mono registerNewAlertsActivityEvents(final List dtos, } private Mono checkDataEntityExistence(final long dataEntityId) { - return dataEntityRepository.exists(dataEntityId).handle((exists, sink) -> { + return dataEntityRepository.existsIncludingSoftDeleted(dataEntityId).handle((exists, sink) -> { if (!exists) { sink.error(new NotFoundException("Data Entity", dataEntityId)); return; diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityFilledServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityFilledServiceImpl.java index 5ca6dec63..45ecbe7dd 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityFilledServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityFilledServiceImpl.java @@ -70,6 +70,7 @@ private boolean needToDelete(final DataEntityFilledPojo pojo) { && !pojo.getDatasetFieldDescriptionFilled() && !pojo.getDatasetFieldLabelsFilled() && !pojo.getDatasetFieldEnumsFilled() - && !pojo.getManuallyCreated(); + && !pojo.getManuallyCreated() + && !pojo.getDatasetFieldTermsFilled(); } } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityGroupService.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityGroupService.java index 2c6d21f82..2fd311a77 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityGroupService.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityGroupService.java @@ -4,7 +4,6 @@ import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityGroupItemList; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRef; import org.opendatadiscovery.oddplatform.ingestion.contract.model.CompactDataEntityList; -import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo; import reactor.core.publisher.Mono; public interface DataEntityGroupService { @@ -12,8 +11,6 @@ public interface DataEntityGroupService { Mono updateDataEntityGroup(final Long id, final DataEntityGroupFormData formData); - Mono deleteDataEntityGroup(final Long id); - Mono listEntitiesWithinDEG(final String degOddrn); Mono listDEGItems(final Long dataEntityGroupId, final Integer page, diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityGroupServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityGroupServiceImpl.java index b4fa83569..a5d8b1155 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityGroupServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityGroupServiceImpl.java @@ -18,7 +18,6 @@ import org.opendatadiscovery.oddplatform.dto.activity.ActivityCreateEvent; import org.opendatadiscovery.oddplatform.dto.activity.ActivityEventTypeDto; import org.opendatadiscovery.oddplatform.exception.BadUserRequestException; -import org.opendatadiscovery.oddplatform.exception.CascadeDeleteException; import org.opendatadiscovery.oddplatform.exception.NotFoundException; import org.opendatadiscovery.oddplatform.ingestion.contract.model.CompactDataEntity; import org.opendatadiscovery.oddplatform.ingestion.contract.model.CompactDataEntityList; @@ -28,18 +27,14 @@ import org.opendatadiscovery.oddplatform.model.tables.pojos.NamespacePojo; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataEntityRepository; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveGroupEntityRelationRepository; -import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveOwnershipRepository; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveSearchEntrypointRepository; -import org.opendatadiscovery.oddplatform.repository.reactive.TermRelationsRepository; import org.opendatadiscovery.oddplatform.service.activity.ActivityLog; import org.opendatadiscovery.oddplatform.service.activity.ActivityParameter; import org.opendatadiscovery.oddplatform.service.activity.ActivityService; -import org.opendatadiscovery.oddplatform.utils.ActivityParameterNames.CustomGroupDeleted; import org.opendatadiscovery.oddplatform.utils.ActivityParameterNames.CustomGroupUpdated; import org.opendatadiscovery.oddrn.Generator; import org.opendatadiscovery.oddrn.model.ODDPlatformDataEntityGroupPath; import org.springframework.stereotype.Service; -import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import static org.opendatadiscovery.oddplatform.dto.DataEntityClassDto.DATA_ENTITY_GROUP; @@ -55,13 +50,10 @@ public class DataEntityGroupServiceImpl implements DataEntityGroupService { private final NamespaceService namespaceService; private final ActivityService activityService; private final DataEntityFilledService dataEntityFilledService; - private final TagService tagService; private final DataEntityStatisticsService dataEntityStatisticsService; private final ReactiveDataEntityRepository reactiveDataEntityRepository; private final ReactiveGroupEntityRelationRepository reactiveGroupEntityRelationRepository; - private final TermRelationsRepository termRelationsRepository; - private final ReactiveOwnershipRepository ownershipRepository; private final ReactiveSearchEntrypointRepository reactiveSearchEntrypointRepository; private final DataEntityMapper dataEntityMapper; @@ -97,21 +89,6 @@ public Mono updateDataEntityGroup( }); } - @Override - @ActivityLog(event = ActivityEventTypeDto.CUSTOM_GROUP_DELETED) - @ReactiveTransactional - public Mono deleteDataEntityGroup( - @ActivityParameter(CustomGroupDeleted.DATA_ENTITY_ID) final Long id) { - return reactiveGroupEntityRelationRepository.degHasEntities(id) - .filter(hasEntities -> !hasEntities) - .switchIfEmpty(Mono.error(new CascadeDeleteException("Can't delete data entity group with entities"))) - .then(reactiveDataEntityRepository.get(id)) - .switchIfEmpty(Mono.error(new NotFoundException("Data entity group", id))) - .filter(DataEntityPojo::getManuallyCreated) - .switchIfEmpty(Mono.error(new BadUserRequestException("Can't delete ingested data entity"))) - .flatMap(this::deleteDEG); - } - @Override public Mono listEntitiesWithinDEG(final String degOddrn) { return reactiveDataEntityRepository.getDEGEntities(degOddrn) @@ -163,7 +140,7 @@ private DataEntityGroupItemList mapToGroupItemsList(final List createDEG(final DataEntityGroupFormData formData, final NamespacePojo namespace) { return Mono.just(formData) - .map(fd -> dataEntityMapper.mapToPojo(fd, DATA_ENTITY_GROUP, namespace)) + .map(fd -> dataEntityMapper.mapCreatedDEGPojo(fd, DATA_ENTITY_GROUP, namespace)) .flatMap(reactiveDataEntityRepository::create) .map(pojo -> { final String oddrn = generateOddrn(pojo); @@ -211,18 +188,6 @@ private Mono updateDEG(final DataEntityPojo pojo, .map(dataEntityMapper::mapRef); } - private Mono deleteDEG(final DataEntityPojo pojo) { - return Flux.zip( - dataEntityStatisticsService.updateStatistics(-1L, - Map.of(DATA_ENTITY_GROUP.getId(), Map.of(pojo.getTypeId(), -1L))), - termRelationsRepository.deleteRelationsWithTerms(pojo.getId()), - reactiveGroupEntityRelationRepository.deleteRelationsForDEG(pojo.getOddrn()), - tagService.deleteRelationsForDataEntity(pojo.getId()), - ownershipRepository.deleteByDataEntityId(pojo.getId()), - dataEntityFilledService.markEntityUnfilled(pojo.getId(), MANUALLY_CREATED) - ).then(reactiveDataEntityRepository.delete(pojo.getId())); - } - private String generateOddrn(final DataEntityPojo pojo) { try { return oddrnGenerator.generate(ODDPlatformDataEntityGroupPath.builder() diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityInternalInformationServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityInternalInformationServiceImpl.java deleted file mode 100644 index da1722d58..000000000 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityInternalInformationServiceImpl.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.opendatadiscovery.oddplatform.service; - -import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.StringUtils; -import org.opendatadiscovery.oddplatform.annotation.ReactiveTransactional; -import org.opendatadiscovery.oddplatform.api.contract.model.InternalDescriptionFormData; -import org.opendatadiscovery.oddplatform.dto.activity.ActivityEventTypeDto; -import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo; -import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataEntityRepository; -import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveSearchEntrypointRepository; -import org.opendatadiscovery.oddplatform.service.activity.ActivityLog; -import org.opendatadiscovery.oddplatform.service.activity.ActivityParameter; -import org.springframework.stereotype.Service; -import reactor.core.publisher.Mono; - -import static org.opendatadiscovery.oddplatform.dto.DataEntityFilledField.INTERNAL_DESCRIPTION; -import static org.opendatadiscovery.oddplatform.utils.ActivityParameterNames.DescriptionUpdated.DATA_ENTITY_ID; - -@Service -@RequiredArgsConstructor -public class DataEntityInternalInformationServiceImpl implements DataEntityInternalInformationService { - private final ReactiveDataEntityRepository reactiveDataEntityRepository; - private final ReactiveSearchEntrypointRepository reactiveSearchEntrypointRepository; - private final DataEntityFilledService dataEntityFilledService; - - @Override - @ReactiveTransactional - @ActivityLog(event = ActivityEventTypeDto.DESCRIPTION_UPDATED) - public Mono updateDescription(@ActivityParameter(DATA_ENTITY_ID) final long dataEntityId, - final InternalDescriptionFormData formData) { - return reactiveDataEntityRepository.setInternalDescription(dataEntityId, formData.getInternalDescription()) - .flatMap(pojo -> reactiveSearchEntrypointRepository.updateDataEntityVectors(dataEntityId) - .thenReturn(pojo)) - .flatMap(pojo -> { - if (StringUtils.isNotEmpty(pojo.getInternalDescription())) { - return dataEntityFilledService.markEntityFilled(dataEntityId, INTERNAL_DESCRIPTION) - .thenReturn(pojo); - } else { - return dataEntityFilledService.markEntityUnfilled(dataEntityId, INTERNAL_DESCRIPTION) - .thenReturn(pojo); - } - }); - } -} diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityInternalInformationService.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityInternalStateService.java similarity index 50% rename from odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityInternalInformationService.java rename to odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityInternalStateService.java index b11660cb3..82eafc54e 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityInternalInformationService.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityInternalStateService.java @@ -1,10 +1,17 @@ package org.opendatadiscovery.oddplatform.service; +import java.util.List; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; import org.opendatadiscovery.oddplatform.api.contract.model.InternalDescriptionFormData; import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo; import reactor.core.publisher.Mono; -public interface DataEntityInternalInformationService { +public interface DataEntityInternalStateService { Mono updateDescription(final long dataEntityId, final InternalDescriptionFormData formData); + + Mono changeStatusForDataEntities(final List pojos, + final DataEntityStatus newStatus); + + Mono restoreDeletedDataEntityRelations(final List deletedPojos); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityInternalStateServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityInternalStateServiceImpl.java new file mode 100644 index 000000000..71a5303bb --- /dev/null +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityInternalStateServiceImpl.java @@ -0,0 +1,188 @@ +package org.opendatadiscovery.oddplatform.service; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.opendatadiscovery.oddplatform.annotation.ReactiveTransactional; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusEnum; +import org.opendatadiscovery.oddplatform.api.contract.model.InternalDescriptionFormData; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; +import org.opendatadiscovery.oddplatform.dto.activity.ActivityContextInfo; +import org.opendatadiscovery.oddplatform.dto.activity.ActivityCreateEvent; +import org.opendatadiscovery.oddplatform.dto.activity.ActivityEventTypeDto; +import org.opendatadiscovery.oddplatform.mapper.DataEntityMapper; +import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityFilledPojo; +import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo; +import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataEntityRepository; +import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveGroupEntityRelationRepository; +import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveGroupParentGroupRelationRepository; +import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveLineageRepository; +import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveSearchEntrypointRepository; +import org.opendatadiscovery.oddplatform.service.activity.ActivityLog; +import org.opendatadiscovery.oddplatform.service.activity.ActivityParameter; +import org.opendatadiscovery.oddplatform.service.activity.ActivityService; +import org.springframework.stereotype.Service; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import static org.apache.commons.lang3.ArrayUtils.contains; +import static org.opendatadiscovery.oddplatform.dto.DataEntityClassDto.DATA_ENTITY_GROUP; +import static org.opendatadiscovery.oddplatform.dto.DataEntityFilledField.INTERNAL_DESCRIPTION; +import static org.opendatadiscovery.oddplatform.dto.DataEntityFilledField.MANUALLY_CREATED; +import static org.opendatadiscovery.oddplatform.utils.ActivityParameterNames.DescriptionUpdated.DATA_ENTITY_ID; +import static org.opendatadiscovery.oddplatform.utils.ActivityParameterNames.StatusUpdated.DATA_ENTITY_POJO; + +@Service +@RequiredArgsConstructor +public class DataEntityInternalStateServiceImpl implements DataEntityInternalStateService { + private final DataEntityFilledService dataEntityFilledService; + private final DataEntityStatisticsService dataEntityStatisticsService; + private final ActivityService activityService; + + private final ReactiveDataEntityRepository dataEntityRepository; + private final ReactiveSearchEntrypointRepository reactiveSearchEntrypointRepository; + private final ReactiveLineageRepository lineageRepository; + private final ReactiveGroupEntityRelationRepository groupEntityRelationRepository; + private final ReactiveGroupParentGroupRelationRepository groupParentGroupRelationRepository; + + private final DataEntityMapper dataEntityMapper; + + @Override + @ReactiveTransactional + @ActivityLog(event = ActivityEventTypeDto.DESCRIPTION_UPDATED) + public Mono updateDescription(@ActivityParameter(DATA_ENTITY_ID) final long dataEntityId, + final InternalDescriptionFormData formData) { + return dataEntityRepository.setInternalDescription(dataEntityId, formData.getInternalDescription()) + .flatMap(pojo -> reactiveSearchEntrypointRepository.updateDataEntityVectors(dataEntityId) + .thenReturn(pojo)) + .flatMap(pojo -> { + if (StringUtils.isNotEmpty(pojo.getInternalDescription())) { + return dataEntityFilledService.markEntityFilled(dataEntityId, INTERNAL_DESCRIPTION) + .thenReturn(pojo); + } else { + return dataEntityFilledService.markEntityUnfilled(dataEntityId, INTERNAL_DESCRIPTION) + .thenReturn(pojo); + } + }); + } + + @Override + @ReactiveTransactional + public Mono changeStatusForDataEntities(final List pojos, + final DataEntityStatus newStatus) { + final List ids = pojos.stream().map(DataEntityPojo::getId).toList(); + if (newStatus.getStatus() == DataEntityStatusEnum.DELETED) { + return getActivityContextInfos(pojos).flatMap(oldStates -> softDeleteDataEntities(pojos) + .then(Mono.defer(() -> logStatusChangeEvents(oldStates, ids)))); + } else { + return getActivityContextInfos(pojos).flatMap(oldStates -> + Flux.fromIterable(pojos) + .collectList() + .flatMap(dataEntityPojos -> { + final List entitiesToRestore = dataEntityPojos.stream() + .filter(pojo -> pojo.getStatus().equals(DataEntityStatusDto.DELETED.getId())) + .toList(); + final List updatedPojos = dataEntityPojos.stream() + .map(pojo -> dataEntityMapper.applyStatus(pojo, newStatus)) + .toList(); + return Mono.when(dataEntityRepository.bulkUpdate(updatedPojos).then(), + restore(entitiesToRestore)); + }) + .then(Mono.defer(() -> logStatusChangeEvents(oldStates, ids))) + ); + } + } + + @Override + @ReactiveTransactional + public Mono restoreDeletedDataEntityRelations(final List deletedPojos) { + return restore(deletedPojos); + } + + private Mono softDeleteDataEntities(final List pojos) { + final List ids = pojos.stream().map(DataEntityPojo::getId).toList(); + final List oddrns = pojos.stream().map(DataEntityPojo::getOddrn).toList(); + return dataEntityRepository.delete(ids) + .then(Mono.defer(() -> { + final Map> statistics = new HashMap<>(); + pojos.forEach(pojo -> Arrays.stream(pojo.getEntityClassIds()).forEach(entityClassId -> { + final Map typesMap = + statistics.computeIfAbsent(entityClassId, id -> new HashMap<>()); + typesMap.merge(pojo.getTypeId(), -1L, Long::sum); + })); + return dataEntityStatisticsService.updateStatistics((long) Math.negateExact(pojos.size()), statistics); + })) + .then(Mono.defer(() -> { + final List> unfilledMonos = pojos.stream() + .filter(this::isManuallyCreatedDEG) + .map(pojo -> dataEntityFilledService.markEntityUnfilled(pojo.getId(), MANUALLY_CREATED)) + .toList(); + return Mono.when(unfilledMonos); + })) + .then(lineageRepository.softDeleteLineageRelations(oddrns).then()) + .then(groupEntityRelationRepository.softDeleteRelationsForDeletedDataEntities(oddrns).then()) + .then(groupParentGroupRelationRepository.softDeleteRelationsForDeletedDataEntities(oddrns).then()); + } + + private Mono restore(final List deletedPojos) { + final List oddrns = deletedPojos.stream().map(DataEntityPojo::getOddrn).toList(); + return lineageRepository.restoreLineageRelations(oddrns) + .then(groupEntityRelationRepository.restoreRelationsForDataEntities(oddrns).then()) + .then(groupParentGroupRelationRepository.restoreRelationsForDataEntities(oddrns).then()) + .then(Mono.defer(() -> { + final Map> statistics = new HashMap<>(); + deletedPojos.forEach(pojo -> Arrays.stream(pojo.getEntityClassIds()).forEach(entityClassId -> { + final Map typesMap = + statistics.computeIfAbsent(entityClassId, id -> new HashMap<>()); + typesMap.merge(pojo.getTypeId(), 1L, Long::sum); + })); + return dataEntityStatisticsService.updateStatistics((long) deletedPojos.size(), statistics); + })) + .then(Mono.defer(() -> { + final List> unfilledMonos = deletedPojos.stream() + .filter(this::isManuallyCreatedDEG) + .map(pojo -> dataEntityFilledService.markEntityFilled(pojo.getId(), MANUALLY_CREATED)) + .toList(); + return Mono.when(unfilledMonos); + })) + .then(); + } + + private Mono> getActivityContextInfos(final List pojos) { + return Flux.fromIterable(pojos) + .flatMap(pojo -> { + final Map parameters = Map.of(DATA_ENTITY_POJO, pojo); + return activityService.getContextInfo(parameters, ActivityEventTypeDto.DATA_ENTITY_STATUS_UPDATED); + }) + .collectList(); + } + + private Mono logStatusChangeEvents(final List oldStates, + final List dataEntityIds) { + return activityService.getUpdatedInfo(Map.of(), dataEntityIds, ActivityEventTypeDto.DATA_ENTITY_STATUS_UPDATED) + .map(updatedInfo -> createStatusUpdatedActivityEvents(oldStates, updatedInfo)) + .flatMap(activityService::createActivityEvents); + } + + private List createStatusUpdatedActivityEvents(final List ctxInfos, + final Map dtoMap) { + return ctxInfos.stream() + .map(ctx -> ActivityCreateEvent.builder() + .dataEntityId(ctx.getDataEntityId()) + .oldState(ctx.getOldState()) + .eventType(ActivityEventTypeDto.DATA_ENTITY_STATUS_UPDATED) + .newState(dtoMap.get(ctx.getDataEntityId())) + .systemEvent(false) + .build()) + .toList(); + } + + private boolean isManuallyCreatedDEG(final DataEntityPojo pojo) { + return pojo.getManuallyCreated() + && contains(pojo.getEntityClassIds(), DATA_ENTITY_GROUP.getId()); + } +} diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityRelationsService.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityRelationsService.java new file mode 100644 index 000000000..d8370d05a --- /dev/null +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityRelationsService.java @@ -0,0 +1,9 @@ +package org.opendatadiscovery.oddplatform.service; + +import java.util.List; +import org.opendatadiscovery.oddplatform.dto.lineage.LineageStreamKind; +import reactor.core.publisher.Mono; + +public interface DataEntityRelationsService { + Mono> getDependentDataEntityOddrns(final LineageStreamKind streamKind); +} diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityRelationsServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityRelationsServiceImpl.java new file mode 100644 index 000000000..b2e03e3d4 --- /dev/null +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityRelationsServiceImpl.java @@ -0,0 +1,40 @@ +package org.opendatadiscovery.oddplatform.service; + +import java.util.List; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.opendatadiscovery.oddplatform.auth.AuthIdentityProvider; +import org.opendatadiscovery.oddplatform.dto.lineage.LineageDepth; +import org.opendatadiscovery.oddplatform.dto.lineage.LineageStreamKind; +import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataEntityRepository; +import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveLineageRepository; +import org.springframework.stereotype.Service; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +@Service +@RequiredArgsConstructor +public class DataEntityRelationsServiceImpl implements DataEntityRelationsService { + private final AuthIdentityProvider authIdentityProvider; + private final ReactiveDataEntityRepository dataEntityRepository; + private final ReactiveLineageRepository lineageRepository; + + @Override + public Mono> getDependentDataEntityOddrns(final LineageStreamKind streamKind) { + return authIdentityProvider.fetchAssociatedOwner() + .flatMapMany(o -> dataEntityRepository.listByOwner(o.getId())) + .map(de -> de.getDataEntity().getOddrn()) + .collect(Collectors.toSet()) + .flatMap(oddrns -> getDependentOddrns(oddrns, streamKind)); + } + + private Mono> getDependentOddrns(final Set oddrns, final LineageStreamKind streamKind) { + return lineageRepository.getLineageRelations(oddrns, LineageDepth.empty(), streamKind) + .flatMap(lp -> Flux.just(lp.getParentOddrn(), lp.getChildOddrn())) + .distinct() + .filter(Predicate.not(oddrns::contains)) + .collectList(); + } +} diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityService.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityService.java index a52b24244..c98659060 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityService.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityService.java @@ -8,6 +8,8 @@ import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityDomainList; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityList; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRef; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusFormData; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityUsageInfo; import org.opendatadiscovery.oddplatform.api.contract.model.DataSourceEntityList; import org.opendatadiscovery.oddplatform.api.contract.model.InternalDescription; @@ -51,8 +53,6 @@ Mono getDataEntitiesByDatasourceAndType(final long datasou Mono getDetails(final long dataEntityId); - Mono> getDependentDataEntityOddrns(final LineageStreamKind streamKind); - Flux listAssociated(final int page, final int size); Flux listAssociated(final int page, final int size, final LineageStreamKind streamKind); @@ -87,4 +87,6 @@ Mono getDataEntityGroupsChildren(final Long dataEntityGroupId, Mono getDataEntityUsageInfo(); Mono getDomainsInfo(); + + Mono updateStatus(final Long dataEntityId, final DataEntityStatusFormData statusFormData); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityServiceImpl.java index 274cd280d..42669cc84 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityServiceImpl.java @@ -21,6 +21,9 @@ import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityDomainList; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityList; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRef; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusEnum; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusFormData; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityUsageInfo; import org.opendatadiscovery.oddplatform.api.contract.model.DataSource; import org.opendatadiscovery.oddplatform.api.contract.model.DataSourceEntityList; @@ -39,6 +42,7 @@ import org.opendatadiscovery.oddplatform.dto.DataEntityClassDto; import org.opendatadiscovery.oddplatform.dto.DataEntityDetailsDto; import org.opendatadiscovery.oddplatform.dto.DataEntityDimensionsDto; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; import org.opendatadiscovery.oddplatform.dto.FacetStateDto; import org.opendatadiscovery.oddplatform.dto.TagDto; import org.opendatadiscovery.oddplatform.dto.activity.ActivityEventTypeDto; @@ -68,7 +72,6 @@ import org.opendatadiscovery.oddplatform.model.tables.pojos.MetadataFieldValuePojo; import org.opendatadiscovery.oddplatform.model.tables.pojos.OwnerPojo; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataEntityRepository; -import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataEntityStatisticsRepository; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataEntityTaskRunRepository; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDatasetVersionRepository; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveGroupEntityRelationRepository; @@ -110,8 +113,10 @@ public class DataEntityServiceImpl implements DataEntityService { private final DataEntityFilledService dataEntityFilledService; private final MetadataFieldService metadataFieldService; private final DataSourceService dataSourceService; + private final DataEntityStatisticsService dataEntityStatisticsService; private final TermService termService; - private final DataEntityInternalInformationService dataEntityInternalInformationService; + private final DataEntityInternalStateService dataEntityInternalStateService; + private final DataEntityRelationsService dataEntityRelationsService; private final ReactiveMetadataFieldValueRepository reactiveMetadataFieldValueRepository; private final ReactiveMetadataFieldRepository reactiveMetadataFieldRepository; @@ -121,7 +126,6 @@ public class DataEntityServiceImpl implements DataEntityService { private final ReactiveDatasetVersionRepository reactiveDatasetVersionRepository; private final ReactiveSearchEntrypointRepository reactiveSearchEntrypointRepository; private final ReactiveGroupEntityRelationRepository reactiveGroupEntityRelationRepository; - private final ReactiveDataEntityStatisticsRepository dataEntityStatisticsRepository; private final ReactiveTagRepository tagRepository; private final DataEntityMapper dataEntityMapper; @@ -210,20 +214,11 @@ public Flux listAssociated(final int page, final int size) { public Flux listAssociated(final int page, final int size, final LineageStreamKind streamKind) { - return this.getDependentDataEntityOddrns(streamKind) - .flatMapMany(oddrns -> reactiveDataEntityRepository.listAllByOddrns(oddrns, false, page, size)) + return dataEntityRelationsService.getDependentDataEntityOddrns(streamKind) + .flatMapMany(oddrns -> reactiveDataEntityRepository.listByOddrns(oddrns, false, false, page, size)) .map(dataEntityMapper::mapRef); } - @Override - public Mono> getDependentDataEntityOddrns(final LineageStreamKind streamKind) { - return authIdentityProvider.fetchAssociatedOwner() - .flatMapMany(o -> reactiveDataEntityRepository.listByOwner(o.getId())) - .map(de -> de.getDataEntity().getOddrn()) - .collect(Collectors.toSet()) - .flatMap(oddrns -> getDependentOddrns(oddrns, streamKind)); - } - @Override public Flux listPopular(final int page, final int size) { return reactiveDataEntityRepository.listPopular(page, size) @@ -323,7 +318,7 @@ public Mono deleteMetadata(final long dataEntityId, final long metadataFie @ReactiveTransactional public Mono upsertDescription(final long dataEntityId, final InternalDescriptionFormData formData) { - return dataEntityInternalInformationService.updateDescription(dataEntityId, formData) + return dataEntityInternalStateService.updateDescription(dataEntityId, formData) .then(termService.handleDataEntityDescriptionTerms(dataEntityId, formData.getInternalDescription())) .map(terms -> { final List linkedTerms = terms.stream().map(termMapper::mapToLinkedTerm).toList(); @@ -438,7 +433,7 @@ public Flux deleteDataEntityFromDEG(final Long dataEnt @Override public Mono getDataEntityUsageInfo() { - return Mono.zip(dataEntityStatisticsRepository.getStatistics(), + return Mono.zip(dataEntityStatisticsService.getStatistics(), dataEntityFilledService.getFilledDataEntitiesCount()) .map(function(dataEntityMapper::mapUsageInfo)); } @@ -454,6 +449,31 @@ public Mono getDomainsInfo() { .map(DataEntityDomainList::new); } + @Override + public Mono updateStatus(final Long dataEntityId, + final DataEntityStatusFormData statusFormData) { + final DataEntityStatus status = statusFormData.getStatus(); + if (isSwitchableStatus(status.getStatus()) && status.getStatusSwitchTime() == null) { + return Mono.error(() -> new BadUserRequestException( + "Status %s must have status switch time".formatted(status.getStatus()))); + } + return reactiveDataEntityRepository.get(dataEntityId) + .switchIfEmpty(Mono.error(() -> new NotFoundException("Data entity", dataEntityId))) + .flatMapMany(pojo -> { + if (needToPropagateStatus(pojo, statusFormData)) { + return reactiveGroupEntityRelationRepository.getDEGEntitiesOddrns(dataEntityId) + .collectList() + .flatMapMany(oddrns -> reactiveDataEntityRepository.listByOddrns(oddrns, false, false)) + .concatWithValues(pojo); + } else { + return Flux.just(pojo); + } + }) + .collectList() + .flatMap(pojos -> dataEntityInternalStateService.changeStatusForDataEntities(pojos, status)) + .thenReturn(statusFormData.getStatus()); + } + private boolean isManuallyCreatedDEG(final DataEntityPojo pojo) { return pojo.getManuallyCreated() && contains(pojo.getEntityClassIds(), DATA_ENTITY_GROUP.getId()); @@ -474,14 +494,6 @@ private Mono> getLastRunsForQualityTests( return reactiveDataEntityTaskRunRepository.getLatestRunsMap(qualityTests); } - private Mono> getDependentOddrns(final Set oddrns, final LineageStreamKind streamKind) { - return reactiveLineageRepository.getLineageRelations(oddrns, LineageDepth.empty(), streamKind) - .flatMap(lp -> Flux.just(lp.getParentOddrn(), lp.getChildOddrn())) - .distinct() - .filter(Predicate.not(oddrns::contains)) - .collectList(); - } - private Set getSpecificAttributesDependentOddrns(final List entities) { return entities.stream() .map(DataEntityDimensionsDto::getSpecificAttributes) @@ -497,7 +509,7 @@ private Mono> enrichEntityClassDetails( final Set dependentOddrns = getSpecificAttributesDependentOddrns(dtos); final Mono> dependencies = reactiveDataEntityRepository - .listAllByOddrns(dependentOddrns, false) + .listByOddrns(dependentOddrns, false, false) .collectMap(DataEntityPojo::getOddrn, identity()); final Mono> lastTaskRuns = getLastRunsForQualityTests(dtos); final Mono>> children = getDEGEntities(dtos); @@ -632,7 +644,7 @@ private Mono> getDEGChildrenCount(final Collection> getConsumersCount(final Collection dtos) { @@ -654,4 +666,14 @@ private Set entityClassOddrns(final Collection .map(dto -> dto.getDataEntity().getOddrn()) .collect(Collectors.toSet()); } + + private boolean isSwitchableStatus(final DataEntityStatusEnum status) { + return DataEntityStatusDto.valueOf(status.name()).isSwitchable(); + } + + private boolean needToPropagateStatus(final DataEntityPojo pojo, + final DataEntityStatusFormData statusFormData) { + return Boolean.TRUE.equals(statusFormData.getPropagate()) + && contains(pojo.getEntityClassIds(), DATA_ENTITY_GROUP.getId()); + } } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityStaleDetector.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityStaleDetector.java new file mode 100644 index 000000000..c70b81eea --- /dev/null +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityStaleDetector.java @@ -0,0 +1,18 @@ +package org.opendatadiscovery.oddplatform.service; + +import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo; +import org.opendatadiscovery.oddplatform.service.ingestion.util.DateTimeUtil; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class DataEntityStaleDetector { + @Value("${odd.data-entity-stale-period}") + private Integer stalePeriod; + + public boolean isDataEntityStale(final DataEntityPojo pojo) { + return pojo.getLastIngestedAt() != null + && stalePeriod != null + && DateTimeUtil.generateNow().isAfter(pojo.getLastIngestedAt().plusDays(stalePeriod)); + } +} diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityStatisticsService.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityStatisticsService.java index 88d340452..89ef5f965 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityStatisticsService.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityStatisticsService.java @@ -7,4 +7,6 @@ public interface DataEntityStatisticsService { Mono updateStatistics(final Long totalDelta, final Map> entityDelta); + + Mono getStatistics(); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityStatisticsServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityStatisticsServiceImpl.java index 4cf7f1b9f..6da53c2c8 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityStatisticsServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataEntityStatisticsServiceImpl.java @@ -29,6 +29,11 @@ public Mono updateStatistics(final Long totalDelta, }); } + @Override + public Mono getStatistics() { + return dataEntityStatisticsRepository.getStatistics(); + } + private Map> getUpdatedCounts(final DataEntityStatisticsPojo existing, final Map> deltaMap) { final Map> existingStatistics = Optional.of(existing) diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataQualityServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataQualityServiceImpl.java index d30c8d02d..f189cb91b 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataQualityServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataQualityServiceImpl.java @@ -19,9 +19,7 @@ import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataQualityRepository; import org.opendatadiscovery.oddplatform.service.sla.SLACalculator; import org.springframework.stereotype.Service; -import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import reactor.function.TupleUtils; import static reactor.function.TupleUtils.consumer; import static reactor.function.TupleUtils.function; @@ -54,10 +52,10 @@ public Mono getDatasetTests(final long datasetId) { @Override public Mono getDatasetTestReport(final long datasetId) { - return reactiveDataEntityRepository.exists(datasetId) + return reactiveDataEntityRepository.existsIncludingSoftDeleted(datasetId) .filter(e -> e) .switchIfEmpty(Mono.error(new NotFoundException("Dataset", datasetId))) - .flatMap(ign -> dataQualityRepository.getDatasetTestReport(datasetId)) + .then(dataQualityRepository.getDatasetTestReport(datasetId)) .map(dataQualityMapper::mapDatasetTestReport); } @@ -66,8 +64,8 @@ public Mono getDatasetTestReport(final long datasetId) { public Mono setDataQualityTestSeverity(final long dataQualityTest, final long datasetId, final DataQualityTestSeverity severity) { - return reactiveDataEntityRepository.exists(datasetId) - .zipWith(reactiveDataEntityRepository.exists(dataQualityTest)) + return reactiveDataEntityRepository.existsIncludingSoftDeleted(datasetId) + .zipWith(reactiveDataEntityRepository.existsIncludingSoftDeleted(dataQualityTest)) .doOnNext(consumer((datasetExists, dqTestExists) -> { if (!datasetExists) { throw new NotFoundException("Dataset", datasetId); @@ -95,7 +93,7 @@ public Mono getSLAReport(final long datasetId) { } private Mono> getDatasetSLA(final long datasetId) { - return reactiveDataEntityRepository.exists(datasetId) + return reactiveDataEntityRepository.existsIncludingSoftDeleted(datasetId) .filter(e -> e) .switchIfEmpty(Mono.error(new NotFoundException("Dataset", datasetId))) .thenMany(dataQualityRepository.getSLA(datasetId)) diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataSourceServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataSourceServiceImpl.java index 769c49b2d..f44b1a340 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataSourceServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataSourceServiceImpl.java @@ -85,7 +85,7 @@ public Mono update(final long id, final DataSourceUpdateFormData for @Override @ReactiveTransactional public Mono delete(final long id) { - return dataEntityRepository.existsByDataSourceId(id) + return dataEntityRepository.existsNonDeletedByDataSourceId(id) .flatMap(exists -> { if (!exists) { return dataSourceRepository.delete(id).map(DataSourcePojo::getId); diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/NamespaceServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/NamespaceServiceImpl.java index 5f2536d55..7fe4e8cd1 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/NamespaceServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/NamespaceServiceImpl.java @@ -1,6 +1,5 @@ package org.opendatadiscovery.oddplatform.service; -import java.util.UUID; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.BooleanUtils; import org.opendatadiscovery.oddplatform.annotation.ReactiveTransactional; @@ -77,7 +76,7 @@ public Mono delete(final long id) { dataSourceRepository.existsByNamespace(id), collectorRepository.existsByNamespace(id), termRepository.existsByNamespace(id), - dataEntityRepository.existsByNamespaceId(id) + dataEntityRepository.existsNonDeletedByNamespaceId(id) ) .map(t -> BooleanUtils.toBoolean(t.getT1()) || BooleanUtils.toBoolean(t.getT2()) diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/OwnershipServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/OwnershipServiceImpl.java index e0eb343f1..c94833c6b 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/OwnershipServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/OwnershipServiceImpl.java @@ -135,7 +135,7 @@ private Mono> propagateOwnership(final OwnershipPojo ownersh final OwnershipPropagateAction action) { return groupEntityRelationRepository.getDEGEntitiesOddrns(ownership.getDataEntityId()) .collectList() - .flatMapMany(childrenOddrns -> dataEntityRepository.listAllByOddrns(childrenOddrns, false)) + .flatMapMany(childrenOddrns -> dataEntityRepository.listByOddrns(childrenOddrns, false, false)) .map(child -> new OwnershipPojo() .setDataEntityId(child.getId()) .setOwnerId(ownership.getOwnerId()) diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/TagService.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/TagService.java index bc87a3c80..4403b3651 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/TagService.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/TagService.java @@ -7,7 +7,6 @@ import org.opendatadiscovery.oddplatform.api.contract.model.TagsResponse; import org.opendatadiscovery.oddplatform.dto.TagDto; import org.opendatadiscovery.oddplatform.model.tables.pojos.TagPojo; -import org.opendatadiscovery.oddplatform.model.tables.pojos.TagToDataEntityPojo; import org.opendatadiscovery.oddplatform.model.tables.pojos.TagToTermPojo; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -29,8 +28,6 @@ public interface TagService { Mono> updateRelationsWithDataEntity(final long dataEntityId, final Set tagNames); - Flux deleteRelationsForDataEntity(final long dataEntityId); - Flux deleteRelationsWithTerm(final long termId, final Set tagsToKeep); diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/TagServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/TagServiceImpl.java index b47532d51..6122b7bd0 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/TagServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/TagServiceImpl.java @@ -29,7 +29,6 @@ @Service @RequiredArgsConstructor public class TagServiceImpl implements TagService { - private final ReactiveTagRepository reactiveTagRepository; private final TagMapper tagMapper; private final ReactiveSearchEntrypointRepository reactiveSearchEntrypointRepository; @@ -121,11 +120,6 @@ public Mono> updateRelationsWithDataEntity(final long dataEntityId, })); } - @Override - public Flux deleteRelationsForDataEntity(final long dataEntityId) { - return reactiveTagRepository.deleteRelationsForDataEntity(dataEntityId); - } - @Override public Flux deleteRelationsWithTerm(final long termId, final Set tagsToKeep) { diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/activity/ActivityServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/activity/ActivityServiceImpl.java index 87e971e85..93257de8c 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/activity/ActivityServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/activity/ActivityServiceImpl.java @@ -21,7 +21,7 @@ import org.opendatadiscovery.oddplatform.mapper.ActivityMapper; import org.opendatadiscovery.oddplatform.model.tables.pojos.ActivityPojo; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveActivityRepository; -import org.opendatadiscovery.oddplatform.service.DataEntityService; +import org.opendatadiscovery.oddplatform.service.DataEntityRelationsService; import org.opendatadiscovery.oddplatform.service.activity.handler.ActivityHandler; import org.opendatadiscovery.oddplatform.service.ingestion.util.DateTimeUtil; import org.springframework.stereotype.Service; @@ -35,7 +35,7 @@ @Slf4j public class ActivityServiceImpl implements ActivityService { private final ReactiveActivityRepository activityRepository; - private final DataEntityService dataEntityService; + private final DataEntityRelationsService dataEntityRelationsService; private final AuthIdentityProvider authIdentityProvider; private final ActivityMapper activityMapper; private final List handlers; @@ -209,7 +209,7 @@ private Flux fetchDependentActivities(final OffsetDateTime beginDate, final Long lastEventId, final OffsetDateTime lastEventDateTime, final LineageStreamKind lineageStreamKind) { - return dataEntityService.getDependentDataEntityOddrns(lineageStreamKind) + return dataEntityRelationsService.getDependentDataEntityOddrns(lineageStreamKind) .flatMapMany(oddrns -> activityRepository.findDependentActivities(beginDate, endDate, size, datasourceId, namespaceId, tagIds, userIds, eventType, oddrns, lastEventId, lastEventDateTime)) .map(activityMapper::mapToActivity) @@ -251,7 +251,7 @@ private Mono getDependentActivitiesCount(final OffsetDateTime beginDate, final List userIds, final ActivityEventTypeDto eventType, final LineageStreamKind lineageStreamKind) { - return dataEntityService.getDependentDataEntityOddrns(lineageStreamKind) + return dataEntityRelationsService.getDependentDataEntityOddrns(lineageStreamKind) .flatMap(oddrns -> activityRepository.getDependentActivitiesCount(beginDate, endDate, datasourceId, namespaceId, tagIds, userIds, eventType, oddrns)) .defaultIfEmpty(0L); diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/activity/handler/CustomGroupDeletedActivityHandler.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/activity/handler/CustomGroupDeletedActivityHandler.java deleted file mode 100644 index ef0632264..000000000 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/activity/handler/CustomGroupDeletedActivityHandler.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.opendatadiscovery.oddplatform.service.activity.handler; - -import java.util.Map; -import org.opendatadiscovery.oddplatform.dto.activity.ActivityContextInfo; -import org.opendatadiscovery.oddplatform.dto.activity.ActivityEventTypeDto; -import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataEntityRepository; -import org.opendatadiscovery.oddplatform.utils.ActivityParameterNames.CustomGroupDeleted; -import org.springframework.stereotype.Component; -import reactor.core.publisher.Mono; - -@Component -public class CustomGroupDeletedActivityHandler extends AbstractCustomGroupActivityHandler implements ActivityHandler { - - public CustomGroupDeletedActivityHandler(final ReactiveDataEntityRepository reactiveDataEntityRepository) { - super(reactiveDataEntityRepository); - } - - @Override - public boolean isHandle(final ActivityEventTypeDto activityEventTypeDto) { - return activityEventTypeDto == ActivityEventTypeDto.CUSTOM_GROUP_DELETED; - } - - @Override - public Mono getContextInfo(final Map parameters) { - final long dataEntityId = (long) parameters.get(CustomGroupDeleted.DATA_ENTITY_ID); - return getCurrentState(dataEntityId) - .map(state -> ActivityContextInfo.builder().oldState(state).dataEntityId(dataEntityId).build()); - } - - @Override - public Mono getUpdatedState(final Map parameters, - final Long dataEntityId) { - return Mono.just("{}"); - } -} diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/activity/handler/DataEntityStatusUpdatedActivityHandler.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/activity/handler/DataEntityStatusUpdatedActivityHandler.java new file mode 100644 index 000000000..1a5f66c4b --- /dev/null +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/activity/handler/DataEntityStatusUpdatedActivityHandler.java @@ -0,0 +1,64 @@ +package org.opendatadiscovery.oddplatform.service.activity.handler; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; +import org.opendatadiscovery.oddplatform.dto.activity.ActivityContextInfo; +import org.opendatadiscovery.oddplatform.dto.activity.ActivityEventTypeDto; +import org.opendatadiscovery.oddplatform.dto.activity.DataEntityStatusUpdatedDto; +import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo; +import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataEntityRepository; +import org.opendatadiscovery.oddplatform.utils.ActivityParameterNames.StatusUpdated; +import org.opendatadiscovery.oddplatform.utils.JSONSerDeUtils; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Mono; + +import static java.util.Collections.singletonList; + +@Component +@RequiredArgsConstructor +public class DataEntityStatusUpdatedActivityHandler implements ActivityHandler { + private final ReactiveDataEntityRepository dataEntityRepository; + + @Override + public boolean isHandle(final ActivityEventTypeDto activityEventTypeDto) { + return activityEventTypeDto == ActivityEventTypeDto.DATA_ENTITY_STATUS_UPDATED; + } + + @Override + public Mono getContextInfo(final Map parameters) { + final DataEntityPojo pojo = (DataEntityPojo) parameters.get(StatusUpdated.DATA_ENTITY_POJO); + return Mono.just(ActivityContextInfo.builder() + .dataEntityId(pojo.getId()) + .oldState(getState(pojo)) + .build()); + } + + @Override + public Mono getUpdatedState(final Map parameters, final Long dataEntityId) { + return getUpdatedState(parameters, singletonList(dataEntityId)) + .handle((map, sink) -> { + final String state = map.get(dataEntityId); + if (state != null) { + sink.next(state); + } + }); + } + + @Override + public Mono> getUpdatedState(final Map parameters, + final List dataEntityIds) { + return dataEntityRepository.get(dataEntityIds) + .collect(Collectors.toMap(DataEntityPojo::getId, this::getState)); + } + + private String getState(final DataEntityPojo pojo) { + final Short statusId = pojo.getStatus(); + final DataEntityStatusDto statusDto = DataEntityStatusDto.findById(statusId) + .orElseThrow(() -> new IllegalArgumentException("Can't find status for id %s".formatted(statusId))); + return JSONSerDeUtils.serializeJson( + new DataEntityStatusUpdatedDto(statusDto.name(), pojo.getStatusSwitchTime())); + } +} diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/attachment/FileUploadService.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/attachment/FileUploadService.java index 7da9fa0c8..28c1e9aac 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/attachment/FileUploadService.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/attachment/FileUploadService.java @@ -1,5 +1,6 @@ package org.opendatadiscovery.oddplatform.service.attachment; +import java.util.List; import java.util.UUID; import org.opendatadiscovery.oddplatform.model.tables.pojos.FilePojo; import org.springframework.core.io.Resource; @@ -12,5 +13,7 @@ public interface FileUploadService { Mono deleteFile(final FilePojo filePojo); + Mono deleteFiles(final List files); + Mono downloadFile(final String path); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/attachment/local/LocalFileUploadServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/attachment/local/LocalFileUploadServiceImpl.java index f64a7d4d2..1b8820a72 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/attachment/local/LocalFileUploadServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/attachment/local/LocalFileUploadServiceImpl.java @@ -3,6 +3,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; import java.util.UUID; import lombok.RequiredArgsConstructor; import org.opendatadiscovery.oddplatform.model.tables.pojos.FilePojo; @@ -58,6 +59,16 @@ public Mono deleteFile(final FilePojo filePojo) { }); } + @Override + public Mono deleteFiles(final List files) { + return blockingOperation(() -> { + for (final FilePojo file : files) { + Files.delete(Paths.get(file.getPath())); + } + return null; + }); + } + @Override public Mono downloadFile(final String path) { final Flux fileChunks = FileUtils.readFile(Paths.get(path)); diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/attachment/remote/RemoteFileUploadServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/attachment/remote/RemoteFileUploadServiceImpl.java index 34813b2bb..4e6a4bb96 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/attachment/remote/RemoteFileUploadServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/attachment/remote/RemoteFileUploadServiceImpl.java @@ -9,6 +9,7 @@ import jakarta.annotation.PostConstruct; import java.io.InputStream; import java.nio.file.Path; +import java.util.List; import java.util.UUID; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; @@ -84,6 +85,15 @@ public Mono deleteFile(final FilePojo filePojo) { return removeObject(args); } + @Override + public Mono deleteFiles(final List files) { + final List> deleteMonos = files.stream() + .map(file -> RemoveObjectArgs.builder().bucket(bucket).object(file.getPath()).build()) + .map(this::removeObject) + .toList(); + return Mono.when(deleteMonos); + } + @Override public Mono downloadFile(final String filePath) { final GetObjectArgs args = GetObjectArgs.builder() diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/ingestion/IngestionServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/ingestion/IngestionServiceImpl.java index c788f5c09..e7675d75f 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/ingestion/IngestionServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/ingestion/IngestionServiceImpl.java @@ -17,6 +17,7 @@ import org.opendatadiscovery.oddplatform.annotation.ReactiveTransactional; import org.opendatadiscovery.oddplatform.dto.DataEntityClassDto; import org.opendatadiscovery.oddplatform.dto.DataEntitySpecificAttributesDelta; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; import org.opendatadiscovery.oddplatform.dto.DataEntityTotalDelta; import org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto; import org.opendatadiscovery.oddplatform.dto.ingestion.DataEntityIngestionDto; @@ -35,6 +36,7 @@ import org.opendatadiscovery.oddplatform.model.tables.pojos.LineagePojo; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataEntityRepository; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataSourceRepository; +import org.opendatadiscovery.oddplatform.service.DataEntityInternalStateService; import org.opendatadiscovery.oddplatform.service.DatasetFieldService; import org.opendatadiscovery.oddplatform.service.ingestion.processor.IngestionProcessorChain; import org.opendatadiscovery.oddplatform.service.metric.OTLPMetricService; @@ -53,6 +55,7 @@ public class IngestionServiceImpl implements IngestionService { private final IngestionProcessorChain ingestionProcessorChain; private final OTLPMetricService otlpMetricService; private final DatasetFieldService datasetFieldService; + private final DataEntityInternalStateService dataEntityInternalStateService; private final ReactiveDataEntityRepository dataEntityRepository; private final ReactiveDataSourceRepository dataSourceRepository; @@ -87,7 +90,7 @@ private Mono persistDataEntities(final long dataSourceId, .map(ingestionMapper::mapTaskRun) .toList(); - return dataEntityRepository.listAllByOddrns(ingestionDtoMap.keySet(), true) + return dataEntityRepository.listByOddrns(ingestionDtoMap.keySet(), true, true) .collect(Collectors.toMap(DataEntityPojo::getOddrn, identity())) .flatMap(existingPojoDict -> { final Map> ingestionDtoPartitions = ingestionDtoMap.values() @@ -113,29 +116,35 @@ private Mono persistDataEntities(final long dataSourceId, .stream() .map(existingDto -> { final DataEntityPojo existingPojo = existingPojoDict.get(existingDto.getOddrn()); - final boolean isEntityUpdated = isEntityUpdated(existingDto, existingPojo); - - return new EnrichedDataEntityIngestionDto(existingPojo.getId(), existingDto, isEntityUpdated); + return new EnrichedDataEntityIngestionDto(existingPojo.getId(), existingPojo, existingDto); }) .toList(); final List entitiesToUpdate = enrichedExistingDtos.stream() - .filter(EnrichedDataEntityIngestionDto::isUpdated) .map(ingestionMapper::dtoToPojo) .toList(); - final List pojosToCreate = ingestionMapper.dtoToPojo(ingestionDtoPartitions.get(false)); + final List entitiesToRestore = enrichedExistingDtos.stream() + .map(EnrichedDataEntityIngestionDto::getPreviousVersionPojo) + .filter(previousVersionPojo -> previousVersionPojo.getStatus() + .equals(DataEntityStatusDto.DELETED.getId())) + .toList(); + final List pojosToCreate = ingestionMapper.dtoToPojo(ingestionDtoPartitions.get(false)); final Flux updated = dataEntityRepository.bulkUpdate(entitiesToUpdate); + final Mono restoredEntities = dataEntityInternalStateService + .restoreDeletedDataEntityRelations(entitiesToRestore); final DataEntityTotalDelta totalDelta = calculateTotalDeltaCount(pojosToCreate, entitiesToUpdate, existingPojoDict); final Flux enrichedNewDtos = dataEntityRepository .bulkCreate(pojosToCreate) - .map(d -> new EnrichedDataEntityIngestionDto(d.getId(), ingestionDtoMap.get(d.getOddrn()))); + .map(d -> new EnrichedDataEntityIngestionDto(d.getId(), null, ingestionDtoMap.get(d.getOddrn()))); - return updated.thenMany(enrichedNewDtos) + return updated + .then(restoredEntities) + .thenMany(enrichedNewDtos) .collectList() .map(newEntities -> buildIngestionRequest(newEntities, enrichedExistingDtos, taskRuns, specificAttributesDeltas, totalDelta)); @@ -194,7 +203,7 @@ private List extractGroupEntityRelations(final Enriche } return dto.getDataEntityGroup().entitiesOddrns().stream() - .map(entityOddrn -> new GroupEntityRelationsPojo(dto.getOddrn(), entityOddrn)) + .map(entityOddrn -> new GroupEntityRelationsPojo(dto.getOddrn(), entityOddrn, false)) .toList(); } @@ -217,7 +226,8 @@ private Optional extractGroupParentGroupRelations return Optional.empty(); } - return Optional.of(new GroupParentGroupRelationsPojo(dto.getOddrn(), dto.getDataEntityGroup().groupOddrn())); + return Optional.of( + new GroupParentGroupRelationsPojo(dto.getOddrn(), dto.getDataEntityGroup().groupOddrn(), false)); } private List extractLineageRelations(final DataEntityIngestionDto dto) { @@ -263,13 +273,6 @@ private List extractLineageRelations(final DataEntityIngestionDto d return result; } - private boolean isEntityUpdated(final DataEntityIngestionDto dto, final DataEntityPojo dePojo) { - return dePojo.getHollow() - || dePojo.getUpdatedAt() == null - || dto.getUpdatedAt() == null - || !dto.getUpdatedAt().equals(dePojo.getUpdatedAt().atOffset(dto.getUpdatedAt().getOffset())); - } - private DataEntityTotalDelta calculateTotalDeltaCount(final List newPojos, final List entitiesToUpdate, final Map existingPojoDict) { diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/ingestion/processor/DatasetStructureIngestionRequestProcessor.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/ingestion/processor/DatasetStructureIngestionRequestProcessor.java index 6106160e5..8e0b0b5a1 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/ingestion/processor/DatasetStructureIngestionRequestProcessor.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/ingestion/processor/DatasetStructureIngestionRequestProcessor.java @@ -83,7 +83,6 @@ private Mono ingestNewDatasetStructure(final IngestionRequest request) { private Mono ingestExistingDatasetStructure(final IngestionRequest request) { final Map datasetDict = request.getExistingEntities().stream() .filter(e -> e.getEntityClasses().contains(DataEntityClassDto.DATA_SET)) - .filter(EnrichedDataEntityIngestionDto::isUpdated) .collect(Collectors.toMap(EnrichedDataEntityIngestionDto::getOddrn, identity())); if (datasetDict.isEmpty()) { diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/job/DataEntityStatusSwitchJob.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/job/DataEntityStatusSwitchJob.java new file mode 100644 index 000000000..168b4c154 --- /dev/null +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/job/DataEntityStatusSwitchJob.java @@ -0,0 +1,32 @@ +package org.opendatadiscovery.oddplatform.service.job; + +import java.util.concurrent.TimeUnit; +import lombok.RequiredArgsConstructor; +import net.javacrumbs.shedlock.core.LockAssert; +import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; +import org.apache.commons.collections4.CollectionUtils; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusEnum; +import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataEntityRepository; +import org.opendatadiscovery.oddplatform.service.DataEntityInternalStateService; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +public class DataEntityStatusSwitchJob { + private final DataEntityInternalStateService internalStateService; + private final ReactiveDataEntityRepository dataEntityRepository; + + @Scheduled(fixedRate = 10, timeUnit = TimeUnit.MINUTES) + @SchedulerLock(name = "statusSwitchJob", lockAtLeastFor = "9m", lockAtMostFor = "9m") + public void run() { + LockAssert.assertLocked(); + final DataEntityStatus status = new DataEntityStatus(DataEntityStatusEnum.DELETED); + dataEntityRepository.getPojosForStatusSwitch() + .collectList() + .filter(CollectionUtils::isNotEmpty) + .flatMap(pojos -> internalStateService.changeStatusForDataEntities(pojos, status)) + .block(); + } +} diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/search/SearchServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/search/SearchServiceImpl.java index 4c3d9d3bf..05473d8f2 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/search/SearchServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/search/SearchServiceImpl.java @@ -168,6 +168,7 @@ private Function>> getFacetFetchOp case OWNERS -> s -> searchFacetRepository.getOwnerFacetForDataEntity(query, page, size, s); case TYPES -> s -> searchFacetRepository.getTypeFacetForDataEntity(query, page, size, s); case GROUPS -> s -> searchFacetRepository.getGroupFacetForDataEntity(query, page, size, s); + case STATUSES -> s -> searchFacetRepository.getStatusFacetForDataEntity(query, page, size, s); }; } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/utils/ActivityParameterNames.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/utils/ActivityParameterNames.java index a009e761c..aa8a55a9b 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/utils/ActivityParameterNames.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/utils/ActivityParameterNames.java @@ -21,6 +21,10 @@ interface InternalNameUpdated { String DATA_ENTITY_ID = "dataEntityId"; } + interface StatusUpdated { + String DATA_ENTITY_POJO = "pojo"; + } + interface TermAssignment { String DATA_ENTITY_ID = "dataEntityId"; } @@ -45,10 +49,6 @@ interface CustomGroupUpdated { String DATA_ENTITY_ID = "dataEntityId"; } - interface CustomGroupDeleted { - String DATA_ENTITY_ID = "dataEntityId"; - } - interface AlertHaltConfigUpdated { String DATA_ENTITY_ID = "dataEntityId"; } diff --git a/odd-platform-api/src/main/resources/application.yml b/odd-platform-api/src/main/resources/application.yml index 883acc8e4..113fecb03 100644 --- a/odd-platform-api/src/main/resources/application.yml +++ b/odd-platform-api/src/main/resources/application.yml @@ -145,6 +145,7 @@ housekeeping: ttl: resolved_alerts_days: 30 search_facets_days: 30 + data_entity_delete_days: 30 notifications: enabled: false @@ -174,6 +175,7 @@ datacollaboration: odd: # platform-base-url: tenant-id: + data-entity-stale-period: 7 # days activity: partition-period: 30 diff --git a/odd-platform-api/src/main/resources/db/migration/V0_0_79__data_deprecation.sql b/odd-platform-api/src/main/resources/db/migration/V0_0_79__data_deprecation.sql new file mode 100644 index 000000000..45f095b0d --- /dev/null +++ b/odd-platform-api/src/main/resources/db/migration/V0_0_79__data_deprecation.sql @@ -0,0 +1,22 @@ +ALTER TABLE data_entity + ADD COLUMN IF NOT EXISTS status SMALLINT DEFAULT 1 NOT NULL; + +UPDATE data_entity +SET status = 5 +WHERE deleted_at IS NOT NULL; + +ALTER TABLE data_entity + ADD COLUMN IF NOT EXISTS status_switch_time TIMESTAMP WITHOUT TIME ZONE; + +ALTER TABLE lineage + ADD COLUMN IF NOT EXISTS is_deleted BOOLEAN NOT NULL DEFAULT FALSE; + +ALTER TABLE group_entity_relations + ADD COLUMN IF NOT EXISTS is_deleted BOOLEAN NOT NULL DEFAULT FALSE; + +ALTER TABLE group_parent_group_relations + ADD COLUMN IF NOT EXISTS is_deleted BOOLEAN NOT NULL DEFAULT FALSE; + +DELETE +FROM activity +WHERE event_type = 'CUSTOM_GROUP_DELETED'; \ No newline at end of file diff --git a/odd-platform-api/src/main/resources/db/migration/V0_0_80__metadata_stale.sql b/odd-platform-api/src/main/resources/db/migration/V0_0_80__metadata_stale.sql new file mode 100644 index 000000000..b388a55ad --- /dev/null +++ b/odd-platform-api/src/main/resources/db/migration/V0_0_80__metadata_stale.sql @@ -0,0 +1,37 @@ +ALTER TABLE data_entity + ADD COLUMN IF NOT EXISTS status_updated_at TIMESTAMP WITHOUT TIME ZONE; + +UPDATE data_entity +SET status_updated_at = deleted_at +WHERE deleted_at IS NOT NULL; + +ALTER TABLE data_entity + ADD COLUMN IF NOT EXISTS platform_created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT (NOW() AT TIME ZONE ('UTC')); + +UPDATE data_entity +SET platform_created_at = created_at +WHERE manually_created = true; + +UPDATE data_entity +SET created_at = NULL +WHERE manually_created = true; + +ALTER TABLE data_entity + RENAME COLUMN created_at TO source_created_at; + +UPDATE data_entity +SET updated_at = NULL +WHERE manually_created = true; + +ALTER TABLE data_entity + RENAME COLUMN updated_at TO source_updated_at; + +ALTER TABLE data_entity + ADD COLUMN IF NOT EXISTS last_ingested_at TIMESTAMP WITHOUT TIME ZONE; + +UPDATE data_entity +SET last_ingested_at = (NOW() AT TIME ZONE ('UTC')) +WHERE manually_created IS FALSE; + +ALTER TABLE data_entity + DROP COLUMN IF EXISTS deleted_at; \ No newline at end of file diff --git a/odd-platform-api/src/main/resources/schema/policy_schema.json b/odd-platform-api/src/main/resources/schema/policy_schema.json index 2b3bc2374..af74ff396 100644 --- a/odd-platform-api/src/main/resources/schema/policy_schema.json +++ b/odd-platform-api/src/main/resources/schema/policy_schema.json @@ -450,8 +450,8 @@ "DATASET_FIELD_ADD_TERM", "DATASET_FIELD_DELETE_TERM", "DATA_ENTITY_GROUP_UPDATE", - "DATA_ENTITY_GROUP_DELETE", - "DATA_ENTITY_ATTACHMENT_MANAGE" + "DATA_ENTITY_ATTACHMENT_MANAGE", + "DATA_ENTITY_STATUS_UPDATE" ] }, "term_permissions": { diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/BaseIngestionTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/BaseIngestionTest.java index 002fae56e..121e989cb 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/BaseIngestionTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/BaseIngestionTest.java @@ -14,6 +14,8 @@ import org.opendatadiscovery.oddplatform.api.contract.model.DataEntity; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityDetails; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRef; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusEnum; import org.opendatadiscovery.oddplatform.api.contract.model.DataSetField; import org.opendatadiscovery.oddplatform.api.contract.model.DataSetStructure; import org.opendatadiscovery.oddplatform.api.contract.model.DataSource; @@ -148,7 +150,7 @@ protected void assertDataEntityDetailsEqual( "dataSource.token", "viewCount", "updatedAt", "versionList", "entityClasses", "type.id", "sourceList", "targetList", "datasetsList", "metadataFieldValues", - "latestRun.id", "latestRun.updatedAt", "latestRun.createdAt" + "latestRun.id", "latestRun.updatedAt", "latestRun.createdAt", "lastIngestedAt" ) .isEqualTo(expected); @@ -244,7 +246,9 @@ protected DataEntityRef buildExpectedDataEntityRef( .externalName(ingestedEntity.getName()) .url("") .hasAlerts(null) - .manuallyCreated(false); + .manuallyCreated(false) + .status(new DataEntityStatus(DataEntityStatusEnum.UNASSIGNED)) + .isStale(false); } private String createSearchId() { diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/DataEntityStatusChangeTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/DataEntityStatusChangeTest.java new file mode 100644 index 000000000..0f27cef65 --- /dev/null +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/DataEntityStatusChangeTest.java @@ -0,0 +1,90 @@ +package org.opendatadiscovery.oddplatform.api; + +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.opendatadiscovery.oddplatform.BaseIngestionTest; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityDetails; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusEnum; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusFormData; +import org.opendatadiscovery.oddplatform.api.contract.model.DataSource; +import org.opendatadiscovery.oddplatform.api.ingestion.utils.IngestionModelGenerator; +import org.opendatadiscovery.oddplatform.ingestion.contract.model.DataEntity; +import org.opendatadiscovery.oddplatform.ingestion.contract.model.DataEntityList; +import org.opendatadiscovery.oddplatform.ingestion.contract.model.DataEntityType; +import org.opendatadiscovery.oddplatform.ingestion.contract.model.DataSet; +import reactor.core.publisher.Mono; + +import static org.assertj.core.api.Assertions.assertThat; + +public class DataEntityStatusChangeTest extends BaseIngestionTest { + + @Test + public void statusChangeTest() { + final DataSource dataSource = createDataSource(); + final DataEntity dataEntity = IngestionModelGenerator.generateSimpleDataEntity(DataEntityType.TABLE) + .dataset(new DataSet()); + final var entityList = new DataEntityList() + .dataSourceOddrn(dataSource.getOddrn()) + .items(List.of(dataEntity)); + + ingestAndAssert(entityList); + + final long foundEntityId = extractIngestedEntityIdAndAssert(dataSource); + changeStatus(foundEntityId, createStatusFormData(false, DataEntityStatusEnum.STABLE, null)); + + final DataEntityDetails details = getDetails(foundEntityId); + assertThat(details.getStatus().getStatus()).isEqualTo(DataEntityStatusEnum.STABLE); + assertThat(details.getStatus().getStatusSwitchTime()).isNull(); + + final OffsetDateTime statusSwitchTime = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); + + changeStatus(foundEntityId, + createStatusFormData(false, DataEntityStatusEnum.DEPRECATED, statusSwitchTime)); + final DataEntityDetails deprecated = getDetails(foundEntityId); + assertThat(deprecated.getStatus().getStatus()).isEqualTo(DataEntityStatusEnum.DEPRECATED); + assertThat(deprecated.getStatus().getStatusSwitchTime()).isEqualTo(statusSwitchTime); + + changeStatusExceptionally(foundEntityId, + createStatusFormData(false, DataEntityStatusEnum.DRAFT, null)); + } + + private void changeStatus(final Long dataEntityId, + final DataEntityStatusFormData status) { + webTestClient.put() + .uri("/api/dataentities/{data_entity_id}/statuses", dataEntityId) + .body(Mono.just(status), DataEntityStatusFormData.class) + .exchange() + .expectStatus().isOk(); + } + + private void changeStatusExceptionally(final Long dataEntityId, + final DataEntityStatusFormData status) { + webTestClient.put() + .uri("/api/dataentities/{data_entity_id}/statuses", dataEntityId) + .body(Mono.just(status), DataEntityStatusFormData.class) + .exchange() + .expectStatus() + .is4xxClientError(); + } + + private DataEntityDetails getDetails(final Long id) { + return webTestClient.get() + .uri("/api/dataentities/{data_entity_id}", id) + .exchange() + .returnResult(DataEntityDetails.class) + .getResponseBody() + .single() + .block(); + } + + private DataEntityStatusFormData createStatusFormData(final boolean isPropagate, + final DataEntityStatusEnum statusEnum, + final OffsetDateTime statusSwitchTime) { + return new DataEntityStatusFormData() + .propagate(isPropagate) + .status(new DataEntityStatus(statusEnum).statusSwitchTime(statusSwitchTime)); + } +} diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/DataEntityStatisticsTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/DataEntityStatisticsTest.java index 46d1f5eb9..2a2d469f6 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/DataEntityStatisticsTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/DataEntityStatisticsTest.java @@ -15,6 +15,9 @@ import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityDetails; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityGroupFormData; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRef; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusEnum; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusFormData; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityTypeUsageInfo; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityUsageInfo; import org.opendatadiscovery.oddplatform.api.contract.model.DataSource; @@ -102,7 +105,13 @@ public void dataEntityStatisticsTest() { .id(DataEntityTypeDto.DAG.getId()) .name(org.opendatadiscovery.oddplatform.api.contract.model.DataEntityType.NameEnum.fromValue( DataEntityTypeDto.DAG.name()))); - formData.setEntities(List.of(new DataEntityRef().id(1L).oddrn(hollowDatasetOddrn))); + formData.setEntities(List.of( + new DataEntityRef() + .id(1L) + .oddrn(hollowDatasetOddrn) + .isStale(false) + .status(new DataEntityStatus(DataEntityStatusEnum.UNASSIGNED)) + )); final DataEntityRef deg = createDEG(formData); @@ -128,7 +137,8 @@ public void dataEntityStatisticsTest() { // Delete DEG final DataEntityDetails details = getDetails(deg.getId()); removeEntityFromDEG(details.getEntities().get(0).getId(), deg.getId()); - deleteDeg(deg.getId()); + changeStatus(deg.getId(), new DataEntityStatusFormData() + .status(new DataEntityStatus(DataEntityStatusEnum.DELETED))); final List afterDEGDeleted = new ArrayList<>(dataEntities); afterDEGDeleted.add(updatedHollow); @@ -172,16 +182,18 @@ private void updateDEG(final Long id, final DataEntityGroupFormData formData) { .expectStatus().isOk(); } - private void deleteDeg(final Long id) { - webTestClient.delete() - .uri("/api/dataentitygroups/{id}", id) + private void changeStatus(final Long dataEntityId, + final DataEntityStatusFormData status) { + webTestClient.put() + .uri("/api/dataentities/{data_entity_id}/statuses", dataEntityId) + .body(Mono.just(status), DataEntityStatusFormData.class) .exchange() - .expectStatus().isNoContent(); + .expectStatus().isOk(); } - private DataEntityDetails getDetails(final Long degId) { + private DataEntityDetails getDetails(final Long id) { return webTestClient.get() - .uri("/api/dataentities/{data_entity_id}", degId) + .uri("/api/dataentities/{data_entity_id}", id) .exchange() .returnResult(DataEntityDetails.class) .getResponseBody() diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/LineageIngestionTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/LineageIngestionTest.java index 2ad147977..7ed494b1b 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/LineageIngestionTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/LineageIngestionTest.java @@ -15,6 +15,9 @@ import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityLineageNode; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityLineageStream; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRef; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusEnum; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusFormData; import org.opendatadiscovery.oddplatform.api.contract.model.DataSetStats; import org.opendatadiscovery.oddplatform.api.contract.model.DataSource; import org.opendatadiscovery.oddplatform.api.ingestion.utils.IngestionModelGenerator; @@ -25,6 +28,8 @@ import org.opendatadiscovery.oddplatform.ingestion.contract.model.DataEntityType; import org.opendatadiscovery.oddplatform.ingestion.contract.model.DataSet; import org.opendatadiscovery.oddplatform.ingestion.contract.model.DataTransformer; +import org.springframework.test.annotation.DirtiesContext; +import reactor.core.publisher.Mono; import static java.util.function.Function.identity; import static org.assertj.core.api.Assertions.assertThat; @@ -42,6 +47,7 @@ public class LineageIngestionTest extends BaseIngestionTest { */ @Test @DisplayName("Simple lineage ingestion test") + @DirtiesContext(methodMode = DirtiesContext.MethodMode.BEFORE_METHOD) public void simpleLineageIngestionTest() { final DataSource createdDataSource = createDataSource(); @@ -239,6 +245,38 @@ public void simpleLineageIngestionTest() { ); assertLineage(ingestedEntities.get(dataTransformer1.getOddrn()), expectedDownstream, expectedUpstream); + + changeStatus(ingestedEntities.get(dataTransformer2.getOddrn()), new DataEntityStatusFormData() + .status(new DataEntityStatus(DataEntityStatusEnum.DELETED))); + + final DataEntityLineage changedExpectedDownstream = new DataEntityLineage() + .root(root) + .downstream( + new DataEntityLineageStream() + .nodes(List.of( + buildExpectedLineageNode( + ingestedEntities.get(dataTransformer1.getOddrn()), + dataTransformer1.getOddrn(), + dataTransformer1.getName(), + createdDataSource, + 3, + 2 + ), + buildExpectedLineageNode( + ingestedEntities.get(middlewareDataset.getOddrn()), + middlewareDataset.getOddrn(), + middlewareDataset.getName(), + createdDataSource, + 1, + 0 + ) + )) + .edges(List.of( + buildExpectedLineageEdge(dataTransformer1, middlewareDataset, ingestedEntities) + )) + ); + + assertLineage(ingestedEntities.get(dataTransformer1.getOddrn()), changedExpectedDownstream, expectedUpstream); } /** @@ -253,6 +291,7 @@ public void simpleLineageIngestionTest() { *

Inner DEG should not be shown in lineage */ @Test + @DirtiesContext(methodMode = DirtiesContext.MethodMode.BEFORE_METHOD) public void simpleDEGLineageIngestionTest() { final DataSource createdDataSource = createDataSource(); final DataEntity inputDataset1 = IngestionModelGenerator.generateSimpleDataEntity(DataEntityType.TABLE) @@ -437,6 +476,15 @@ private void assertLineage(final long dataEntityId, }); } + private void changeStatus(final Long dataEntityId, + final DataEntityStatusFormData status) { + webTestClient.put() + .uri("/api/dataentities/{data_entity_id}/statuses", dataEntityId) + .body(Mono.just(status), DataEntityStatusFormData.class) + .exchange() + .expectStatus().isOk(); + } + private DataEntityLineageEdge buildExpectedLineageEdge(final DataEntity source, final DataEntity target, final Map idMap) { @@ -464,6 +512,8 @@ private DataEntityLineageNode buildExpectedLineageNode(final long id, .externalName(name) .dataSource(dataSource) .parentsCount(parentsCount) - .childrenCount(childrenCount); + .childrenCount(childrenCount) + .status(new DataEntityStatus(DataEntityStatusEnum.UNASSIGNED)) + .isStale(false); } } diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/utils/IngestionModelMapper.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/utils/IngestionModelMapper.java index 5e46fb897..aa8f8c058 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/utils/IngestionModelMapper.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/utils/IngestionModelMapper.java @@ -8,6 +8,8 @@ import org.opendatadiscovery.oddplatform.api.contract.model.BooleanFieldStat; import org.opendatadiscovery.oddplatform.api.contract.model.ComplexFieldStat; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityDetails; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusEnum; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityType; import org.opendatadiscovery.oddplatform.api.contract.model.DataSetField; import org.opendatadiscovery.oddplatform.api.contract.model.DataSetFieldStat; @@ -35,7 +37,7 @@ public static DataEntityDetails buildExpectedBaseDEDetails(final long entityId, final DataSource dataSource) { return new DataEntityDetails() .id(entityId) - .createdAt(dataEntity.getCreatedAt()) + .sourceCreatedAt(dataEntity.getCreatedAt()) .dataSource(dataSource) .externalName(dataEntity.getName()) .oddrn(dataEntity.getOddrn()) @@ -43,6 +45,8 @@ public static DataEntityDetails buildExpectedBaseDEDetails(final long entityId, .dataEntityGroups(emptyList()) .tags(emptyList()) .terms(emptyList()) + .status(new DataEntityStatus().status(DataEntityStatusEnum.UNASSIGNED)) + .isStale(false) .type(new DataEntityType().name(DataEntityType.NameEnum.fromValue(dataEntity.getType().getValue()))); } diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/ActivityMapperTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/ActivityMapperTest.java index d9a518e91..fe896cbef 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/ActivityMapperTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/ActivityMapperTest.java @@ -26,6 +26,7 @@ import org.opendatadiscovery.oddplatform.api.contract.model.OwnershipActivityState; import org.opendatadiscovery.oddplatform.dto.AssociatedOwnerDto; import org.opendatadiscovery.oddplatform.dto.DataEntityClassDto; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; import org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto; import org.opendatadiscovery.oddplatform.dto.activity.ActivityCreateEvent; import org.opendatadiscovery.oddplatform.dto.activity.ActivityDto; @@ -36,6 +37,7 @@ import org.opendatadiscovery.oddplatform.dto.activity.BusinessNameActivityStateDto; import org.opendatadiscovery.oddplatform.dto.activity.CustomGroupActivityStateDto; import org.opendatadiscovery.oddplatform.dto.activity.DataEntityCreatedActivityStateDto; +import org.opendatadiscovery.oddplatform.dto.activity.DataEntityStatusUpdatedDto; import org.opendatadiscovery.oddplatform.dto.activity.DatasetFieldEnumValuesActivityStateDto; import org.opendatadiscovery.oddplatform.dto.activity.DatasetFieldInformationActivityStateDto; import org.opendatadiscovery.oddplatform.dto.activity.DatasetFieldLabelActivityStateDto; @@ -48,6 +50,7 @@ import org.opendatadiscovery.oddplatform.model.tables.pojos.ActivityPojo; import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo; import org.opendatadiscovery.oddplatform.model.tables.pojos.OwnerPojo; +import org.opendatadiscovery.oddplatform.service.DataEntityStaleDetector; import org.opendatadiscovery.oddplatform.service.ingestion.util.DateTimeUtil; import org.opendatadiscovery.oddplatform.utils.JSONSerDeUtils; import org.opendatadiscovery.oddplatform.utils.RecordFactory; @@ -74,6 +77,9 @@ public class ActivityMapperTest { @InjectMocks ActivityMapper activityMapper = new ActivityMapperImpl(new DateTimeMapperImpl()); + DataEntityStatusMapper statusMapper = new DataEntityStatusMapper(); + DataEntityStaleDetector staleDetector = new DataEntityStaleDetector(); + @ParameterizedTest @EnumSource(ActivityEventTypeDto.class) @DisplayName("Test mapping activity create event to activity pojo object") @@ -101,7 +107,7 @@ void testMapToActivity(final ActivityEventTypeDto eventTypeDto) { .setName(UUID.randomUUID().toString()); final ActivityPojo activityPojo = createPojo(eventTypeDto); final DataEntityPojo dataEntityPojo = createDataEntityPojo(activityPojo.getDataEntityId()); - final DataEntityType dataEntityType = new DataEntityType().id(1).name(DataEntityType.NameEnum.TABLE); + final DataEntityType dataEntityType = new DataEntityType(1, DataEntityType.NameEnum.TABLE); final DataEntityClass dataEntityClass = new DataEntityClass() .id(1) .name(DataEntityClass.NameEnum.SET); @@ -125,6 +131,8 @@ void testMapToActivity(final ActivityEventTypeDto eventTypeDto) { .internalName(dataEntityPojo.getInternalName()) .entityClasses(List.of(dataEntityClass)) .manuallyCreated(dataEntityPojo.getManuallyCreated()) + .status(statusMapper.mapStatus(dataEntityPojo)) + .isStale(staleDetector.isDataEntityStale(dataEntityPojo)) .url(""); lenient().when(dataEntityMapper.mapRef(dataEntityPojo)).thenReturn(ref); @@ -188,7 +196,8 @@ private DataEntityPojo createDataEntityPojo(final Long id) { .setEntityClassIds(new Integer[] {1, 2, 3}) .setTypeId(1) .setExternalName(UUID.randomUUID().toString()) - .setOddrn(UUID.randomUUID().toString()); + .setOddrn(UUID.randomUUID().toString()) + .setStatus((short) 1); } private ActivityCreateEvent createEvent(final ActivityEventTypeDto eventTypeDto) { @@ -212,15 +221,24 @@ private String generateState(final ActivityEventTypeDto eventTypeDto) { case DATASET_FIELD_VALUES_UPDATED -> generateDatasetFieldValuesState(); case DATASET_FIELD_DESCRIPTION_UPDATED, DATASET_FIELD_LABELS_UPDATED -> generateDatasetFieldInformationState(); - case CUSTOM_GROUP_CREATED, CUSTOM_GROUP_UPDATED, CUSTOM_GROUP_DELETED -> generateCustomGroupState(); + case CUSTOM_GROUP_CREATED, CUSTOM_GROUP_UPDATED -> generateCustomGroupState(); case ALERT_HALT_CONFIG_UPDATED -> generateAlertHaltConfigState(); case ALERT_STATUS_UPDATED -> generateAlertStatusState(); case OPEN_ALERT_RECEIVED, RESOLVED_ALERT_RECEIVED -> generateAlertReceivedState(); case DATASET_FIELD_TERM_ASSIGNMENT_UPDATED -> generateDatasetFieldTermsState(); + case DATA_ENTITY_STATUS_UPDATED -> generateDataEntityStatusState(); default -> ""; }; } + private String generateDataEntityStatusState() { + final DataEntityStatusUpdatedDto state = new DataEntityStatusUpdatedDto( + DataEntityStatusDto.STABLE.name(), + DateTimeUtil.generateNow() + ); + return JSONSerDeUtils.serializeJson(state); + } + private String generateAlertReceivedState() { final AlertReceivedActivityStateDto state = GENERATOR.nextObject(AlertReceivedActivityStateDto.class); return JSONSerDeUtils.serializeJson(state); diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/DatasetVersionMapperTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/DatasetVersionMapperTest.java index 91e334fd9..8a06a9218 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/DatasetVersionMapperTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/DatasetVersionMapperTest.java @@ -105,7 +105,7 @@ void testMapNewDatasetVersion() { "parentOddrn", null, "structHash", 1L)); final long expectedVersion = 1L; final EnrichedDataEntityIngestionDto dto = - new EnrichedDataEntityIngestionDto(expectedVersion, dataEntityIngestionDto); + new EnrichedDataEntityIngestionDto(expectedVersion, null, dataEntityIngestionDto); final DatasetVersionPojo actualDatasetVersionPojo = datasetVersionMapper.mapDatasetVersion(dto.getOddrn(), dto.getDataSet().structureHash(), expectedVersion); diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/LineageMapperTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/LineageMapperTest.java index 879502b47..89bad0ef5 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/LineageMapperTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/LineageMapperTest.java @@ -14,6 +14,7 @@ import org.opendatadiscovery.oddplatform.dto.lineage.DataEntityLineageDto; import org.opendatadiscovery.oddplatform.dto.lineage.DataEntityLineageStreamDto; import org.opendatadiscovery.oddplatform.dto.lineage.LineageNodeDto; +import org.opendatadiscovery.oddplatform.service.DataEntityStaleDetector; import org.opendatadiscovery.oddplatform.utils.Pair; import org.opendatadiscovery.oddplatform.utils.RecordFactory; @@ -27,13 +28,14 @@ class LineageMapperTest { private static final EasyRandom EASY_RANDOM; static { - final EasyRandomParameters EASY_RANDOMParameters = new EasyRandomParameters() + final EasyRandomParameters params = new EasyRandomParameters() .scanClasspathForConcreteTypes(true) .excludeField(named("hasChildren")) + .randomize(f -> f.getName().equals("status") && f.getType().isAssignableFrom(Short.class), () -> (short) 1) .randomizationDepth(10) .objectFactory(new RecordFactory()); - EASY_RANDOM = new EasyRandom(EASY_RANDOMParameters); + EASY_RANDOM = new EasyRandom(params); } @BeforeEach @@ -73,7 +75,9 @@ void setUp() { new DateTimeMapperImpl() ), termMapper, - new DateTimeMapperImpl() + new DateTimeMapperImpl(), + new DataEntityStatusMapper(), + new DataEntityStaleDetector() ) ); mapper.setDataSourceMapper( @@ -82,6 +86,8 @@ void setUp() { new TokenMapperImpl(new DateTimeMapperImpl()) ) ); + mapper.setDataEntityStaleDetector(new DataEntityStaleDetector()); + mapper.setDataEntityStatusMapper(new DataEntityStatusMapper()); } @Test diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/repository/DataQualityRepositoryImplTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/repository/DataQualityRepositoryImplTest.java index b37d5fb40..53cf95a2a 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/repository/DataQualityRepositoryImplTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/repository/DataQualityRepositoryImplTest.java @@ -54,16 +54,12 @@ class DataQualityRepositoryImplTest extends BaseIntegrationTest { @Test public void testGetDataQualityTestOddrnsForHollowDataset() { final DataEntityPojo hollowDataEntity = dataEntityRepository - .bulkCreate(List.of(new DataEntityPojo().setHollow(true).setOddrn(UUID.randomUUID().toString()))) - .collectList() - .block() - .get(0); + .create(new DataEntityPojo().setHollow(true).setOddrn(UUID.randomUUID().toString())) + .block(); final DataEntityPojo dqTest = dataEntityRepository - .bulkCreate(List.of(new DataEntityPojo().setOddrn(UUID.randomUUID().toString()))) - .collectList() - .block() - .get(0); + .create(new DataEntityPojo().setOddrn(UUID.randomUUID().toString())) + .block(); dataQualityTestRelationRepository.createRelations(List.of( new DataQualityTestRelationsPojo() @@ -200,7 +196,12 @@ public void testGetDatasetTestReport() { .setDatasetOddrn(dataEntity.getOddrn())) .toList(); - final DatasetTestReportDto expected = mapReport(taskRuns.values().stream() + final DataEntityPojo dqTestToDelete = dqTests.get(0); + dataEntityRepository.delete(dqTestToDelete.getId()).block(); + + final DatasetTestReportDto expected = mapReport(taskRuns.entrySet().stream() + .filter(e -> !e.getKey().equals(dqTestToDelete.getOddrn())) + .map(Map.Entry::getValue) .map(dataEntityTaskRunPojos -> dataEntityTaskRunPojos .stream() .min(endTimeComparator) diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/repository/LineageRepositoryTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/repository/LineageRepositoryTest.java index 496f3d903..7b41811ee 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/repository/LineageRepositoryTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/repository/LineageRepositoryTest.java @@ -8,6 +8,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.RandomStringUtils; +import org.assertj.core.api.Assertions; import org.jeasy.random.EasyRandom; import org.jeasy.random.EasyRandomParameters; import org.junit.jupiter.api.Test; @@ -30,12 +31,12 @@ class LineageRepositoryTest extends BaseIntegrationTest { private static final EasyRandom EASY_RANDOM; static { - final EasyRandomParameters EASY_RANDOMParameters = new EasyRandomParameters() + final EasyRandomParameters params = new EasyRandomParameters() .scanClasspathForConcreteTypes(true) .randomizationDepth(10) .objectFactory(new RecordFactory()); - EASY_RANDOM = new EasyRandom(EASY_RANDOMParameters); + EASY_RANDOM = new EasyRandom(params); } @Autowired @@ -53,7 +54,7 @@ void batchDeleteByEstablisherOddrnTest() { final var secondPojoToDelete = EASY_RANDOM.nextObject(LineagePojo.class).setEstablisherOddrn(secondEstablisherOddrnToDelete); final var pojoToKeep = EASY_RANDOM.nextObject(LineagePojo.class).setEstablisherOddrn(establisherOddrnToKeep); - lineageRepository.bulkCreate(List.of(firstPojoToDelete, secondPojoToDelete, pojoToKeep)).blockLast(); + lineageRepository.batchInsertLineages(List.of(firstPojoToDelete, secondPojoToDelete, pojoToKeep)).blockLast(); lineageRepository.batchDeleteByEstablisherOddrn( List.of(firstEstablisherOddrnToDelete, secondEstablisherOddrnToDelete)) .as(StepVerifier::create) @@ -67,11 +68,15 @@ void batchInsertLineagesTest() { final var firstPojoToInsert = EASY_RANDOM.nextObject(LineagePojo.class); final var secondPojoToInsert = EASY_RANDOM.nextObject(LineagePojo.class); final var duplicatedPojo = EASY_RANDOM.nextObject(LineagePojo.class); - lineageRepository.create(duplicatedPojo).block(); + lineageRepository.batchInsertLineages(List.of(duplicatedPojo)).blockFirst(); lineageRepository.batchInsertLineages(List.of(firstPojoToInsert, secondPojoToInsert, duplicatedPojo)) .as(StepVerifier::create) - .expectNext(firstPojoToInsert) - .expectNext(secondPojoToInsert) + .assertNext(pojo -> assertThat(pojo).usingRecursiveComparison() + .ignoringFields("isDeleted") + .isEqualTo(firstPojoToInsert)) + .assertNext(pojo -> assertThat(pojo).usingRecursiveComparison() + .ignoringFields("isDeleted") + .isEqualTo(secondPojoToInsert)) .verifyComplete(); } @@ -95,7 +100,8 @@ void getTargetsCountTest() { .setChildOddrn(RandomStringUtils.randomAlphabetic(5)) .setEstablisherOddrn(RandomStringUtils.randomAlphabetic(5)); dataEntityRepository.create(pojo).block(); - lineageRepository.bulkCreate(List.of(firstLineagePojo, secondLineagePojo, notCountedLineagePojo)).blockLast(); + lineageRepository.batchInsertLineages(List.of(firstLineagePojo, secondLineagePojo, notCountedLineagePojo)) + .blockLast(); lineageRepository.getTargetsCount(Set.of(parentOddrn)) .as(StepVerifier::create) .assertNext(r -> assertThat(r.get(parentOddrn)).isEqualTo(2L)) @@ -111,7 +117,7 @@ void getLineageRelationsTest_WithOddrns() { .setChildOddrn(expectedChildOddrn) .setEstablisherOddrn(RandomStringUtils.randomAlphabetic(5)); final var excludedLineage = EASY_RANDOM.nextObject(LineagePojo.class); - lineageRepository.bulkCreate(List.of(expectedLineage, excludedLineage)).blockLast(); + lineageRepository.batchInsertLineages(List.of(expectedLineage, excludedLineage)).blockLast(); lineageRepository.getLineageRelations(List.of(expectedChildOddrn, expectedParentOddrn)) .as(StepVerifier::create) .assertNext(r -> assertThat(r) @@ -132,27 +138,40 @@ void getLineageRelationsTest_WithRoots() { final var firstRootFirstChildFirstChildFirstChildOddrn = "firstRootFirstChildFirstChildFirstChildOddrn"; final var secondRootFirstChildOddrn = "secondRootFirstChildOddrn"; - final var firstRootFirstChildLineage = - new LineagePojo(firstRootOddrn, firstRootFirstChildOddrn, RandomStringUtils.randomAlphabetic(5)); - final var firstRootSecondChildLineage = - new LineagePojo(firstRootOddrn, firstRootSecondChildOddrn, RandomStringUtils.randomAlphabetic(5)); - final var firstRootFirstChildFirstChildLineage = - new LineagePojo(firstRootFirstChildOddrn, firstRootFirstChildFirstChildOddrn, - RandomStringUtils.randomAlphabetic(5)); - final var firstRootFirstChildSecondChildLineage = - new LineagePojo(firstRootFirstChildOddrn, firstRootFirstChildSecondChildOddrn, - RandomStringUtils.randomAlphabetic(5)); - final var firstRootFirstChildFirstChildFirstChildLineage = - new LineagePojo(firstRootFirstChildFirstChildOddrn, firstRootFirstChildFirstChildFirstChildOddrn, - RandomStringUtils.randomAlphabetic(5)); - final var secondRootFirstChildLineage = - new LineagePojo(secondRootOddrn, secondRootFirstChildOddrn, RandomStringUtils.randomAlphabetic(5)); + final var firstRootFirstChildLineage = new LineagePojo() + .setParentOddrn(firstRootOddrn) + .setChildOddrn(firstRootFirstChildOddrn) + .setEstablisherOddrn(RandomStringUtils.randomAlphabetic(5)); + final var firstRootSecondChildLineage = new LineagePojo() + .setParentOddrn(firstRootOddrn) + .setChildOddrn(firstRootSecondChildOddrn) + .setEstablisherOddrn(RandomStringUtils.randomAlphabetic(5)); + final var firstRootFirstChildFirstChildLineage = new LineagePojo() + .setParentOddrn(firstRootFirstChildOddrn) + .setChildOddrn(firstRootFirstChildFirstChildOddrn) + .setEstablisherOddrn(RandomStringUtils.randomAlphabetic(5)); + final var firstRootFirstChildSecondChildLineage = new LineagePojo() + .setParentOddrn(firstRootFirstChildOddrn) + .setChildOddrn(firstRootFirstChildSecondChildOddrn) + .setEstablisherOddrn(RandomStringUtils.randomAlphabetic(5)); + final var firstRootFirstChildFirstChildFirstChildLineage = new LineagePojo() + .setParentOddrn(firstRootFirstChildFirstChildOddrn) + .setChildOddrn(firstRootFirstChildFirstChildFirstChildOddrn) + .setEstablisherOddrn(RandomStringUtils.randomAlphabetic(5)); + final var secondRootFirstChildLineage = new LineagePojo() + .setParentOddrn(secondRootOddrn) + .setChildOddrn(secondRootFirstChildOddrn) + .setEstablisherOddrn(RandomStringUtils.randomAlphabetic(5)); final var randomLineage = generateLineageWithParent(RandomStringUtils.randomAlphabetic(5)); - lineageRepository.bulkCreate( - List.of(firstRootFirstChildLineage, firstRootSecondChildLineage, firstRootFirstChildFirstChildLineage, - firstRootFirstChildSecondChildLineage, firstRootFirstChildFirstChildFirstChildLineage, - secondRootFirstChildLineage, randomLineage)).blockLast(); + lineageRepository.batchInsertLineages(List.of( + firstRootFirstChildLineage, + firstRootSecondChildLineage, + firstRootFirstChildFirstChildLineage, + firstRootFirstChildSecondChildLineage, + firstRootFirstChildFirstChildFirstChildLineage, + secondRootFirstChildLineage, randomLineage) + ).blockLast(); final var expectedDownstreamWithDepth3 = Stream.of( firstRootFirstChildLineage, @@ -161,7 +180,6 @@ void getLineageRelationsTest_WithRoots() { firstRootFirstChildSecondChildLineage, firstRootFirstChildFirstChildFirstChildLineage, secondRootFirstChildLineage) - .map(l -> new LineagePojo(l.getParentOddrn(), l.getChildOddrn(), null)) .collect(Collectors.toSet()); final var expectedDownstreamWithDepth2 = Stream.of( @@ -170,59 +188,56 @@ void getLineageRelationsTest_WithRoots() { firstRootFirstChildFirstChildLineage, firstRootFirstChildSecondChildLineage, secondRootFirstChildLineage) - .map(l -> new LineagePojo(l.getParentOddrn(), l.getChildOddrn(), null)) .collect(Collectors.toSet()); final var expectedDownstreamWithDepth1 = Stream.of( firstRootFirstChildLineage, firstRootSecondChildLineage, secondRootFirstChildLineage) - .map(l -> new LineagePojo(l.getParentOddrn(), l.getChildOddrn(), null)) .collect(Collectors.toSet()); final var expectedUpstreamWithDepth3 = Stream.of( firstRootFirstChildFirstChildFirstChildLineage, firstRootFirstChildFirstChildLineage, - firstRootFirstChildLineage - ).map(l -> new LineagePojo(l.getParentOddrn(), l.getChildOddrn(), null)) + firstRootFirstChildLineage) .collect(Collectors.toSet()); final var expectedUpstreamWithDepth2 = Stream.of( firstRootFirstChildFirstChildFirstChildLineage, - firstRootFirstChildFirstChildLineage - ).map(l -> new LineagePojo(l.getParentOddrn(), l.getChildOddrn(), null)) + firstRootFirstChildFirstChildLineage) .collect(Collectors.toSet()); - final var expectedUpstreamWithDepth1 = Stream.of( - firstRootFirstChildFirstChildFirstChildLineage) - .map(l -> new LineagePojo(l.getParentOddrn(), l.getChildOddrn(), null)) + final var expectedUpstreamWithDepth1 = Stream.of(firstRootFirstChildFirstChildFirstChildLineage) .collect(Collectors.toSet()); lineageRepository.getLineageRelations(Set.of(firstRootOddrn, secondRootOddrn), LineageDepth.of(3), LineageStreamKind.DOWNSTREAM) + .collectList() .as(StepVerifier::create) - .recordWith(HashSet::new) - .thenConsumeWhile(r -> true) - .expectRecordedMatches(lineages -> lineages.equals(expectedDownstreamWithDepth3)) + .assertNext(lineages -> Assertions.assertThat(lineages) + .usingRecursiveFieldByFieldElementComparatorIgnoringFields("establisherOddrn", "isDeleted") + .hasSameElementsAs(expectedDownstreamWithDepth3)) .verifyComplete(); lineageRepository.getLineageRelations(Set.of(firstRootOddrn, secondRootOddrn), LineageDepth.of(2), LineageStreamKind.DOWNSTREAM) + .collectList() .as(StepVerifier::create) - .recordWith(HashSet::new) - .thenConsumeWhile(r -> true) - .expectRecordedMatches(lineages -> lineages.equals(expectedDownstreamWithDepth2)) + .assertNext(lineages -> Assertions.assertThat(lineages) + .usingRecursiveFieldByFieldElementComparatorIgnoringFields("establisherOddrn", "isDeleted") + .hasSameElementsAs(expectedDownstreamWithDepth2)) .verifyComplete(); lineageRepository.getLineageRelations(Set.of(firstRootOddrn, secondRootOddrn), LineageDepth.of(1), LineageStreamKind.DOWNSTREAM) + .collectList() .as(StepVerifier::create) - .recordWith(HashSet::new) - .thenConsumeWhile(r -> true) - .expectRecordedMatches(lineages -> lineages.equals(expectedDownstreamWithDepth1)) + .assertNext(lineages -> Assertions.assertThat(lineages) + .usingRecursiveFieldByFieldElementComparatorIgnoringFields("establisherOddrn", "isDeleted") + .hasSameElementsAs(expectedDownstreamWithDepth1)) .verifyComplete(); lineageRepository.getLineageRelations(Set.of(firstRootOddrn, secondRootOddrn), @@ -237,28 +252,31 @@ void getLineageRelationsTest_WithRoots() { lineageRepository.getLineageRelations(Set.of(firstRootFirstChildFirstChildFirstChildOddrn), LineageDepth.of(3), LineageStreamKind.UPSTREAM) + .collectList() .as(StepVerifier::create) - .recordWith(HashSet::new) - .thenConsumeWhile(r -> true) - .expectRecordedMatches(lineages -> lineages.equals(expectedUpstreamWithDepth3)) + .assertNext(lineages -> Assertions.assertThat(lineages) + .usingRecursiveFieldByFieldElementComparatorIgnoringFields("establisherOddrn", "isDeleted") + .hasSameElementsAs(expectedUpstreamWithDepth3)) .verifyComplete(); lineageRepository.getLineageRelations(Set.of(firstRootFirstChildFirstChildFirstChildOddrn), LineageDepth.of(2), LineageStreamKind.UPSTREAM) + .collectList() .as(StepVerifier::create) - .recordWith(HashSet::new) - .thenConsumeWhile(r -> true) - .expectRecordedMatches(lineages -> lineages.equals(expectedUpstreamWithDepth2)) + .assertNext(lineages -> Assertions.assertThat(lineages) + .usingRecursiveFieldByFieldElementComparatorIgnoringFields("establisherOddrn", "isDeleted") + .hasSameElementsAs(expectedUpstreamWithDepth2)) .verifyComplete(); lineageRepository.getLineageRelations(Set.of(firstRootFirstChildFirstChildFirstChildOddrn), LineageDepth.of(1), LineageStreamKind.UPSTREAM) + .collectList() .as(StepVerifier::create) - .recordWith(HashSet::new) - .thenConsumeWhile(r -> true) - .expectRecordedMatches(lineages -> lineages.equals(expectedUpstreamWithDepth1)) + .assertNext(lineages -> Assertions.assertThat(lineages) + .usingRecursiveFieldByFieldElementComparatorIgnoringFields("establisherOddrn", "isDeleted") + .hasSameElementsAs(expectedUpstreamWithDepth1)) .verifyComplete(); } @@ -268,7 +286,7 @@ void getChildrenCountTest() { final var secondParentOddrn = RandomStringUtils.randomAlphabetic(5); final var expected = Map.of(firstParentOddrn, 2, secondParentOddrn, 1); - lineageRepository.bulkCreate(List.of(generateLineageWithParent(firstParentOddrn), + lineageRepository.batchInsertLineages(List.of(generateLineageWithParent(firstParentOddrn), generateLineageWithParent(firstParentOddrn), generateLineageWithParent(secondParentOddrn))).blockLast(); @@ -283,7 +301,7 @@ void getParentCountTest() { final var firstChildOddrn = RandomStringUtils.randomAlphabetic(5); final var secondChildOddrn = RandomStringUtils.randomAlphabetic(5); final var expected = Map.of(firstChildOddrn, 2, secondChildOddrn, 1); - lineageRepository.bulkCreate(List.of(generateLineageWithChild(firstChildOddrn), + lineageRepository.batchInsertLineages(List.of(generateLineageWithChild(firstChildOddrn), generateLineageWithChild(firstChildOddrn), generateLineageWithChild(secondChildOddrn))).blockLast(); lineageRepository.getParentCount(Set.of(firstChildOddrn, secondChildOddrn)) @@ -293,14 +311,16 @@ void getParentCountTest() { } private LineagePojo generateLineageWithParent(final String parentOddrn) { - return new LineagePojo(parentOddrn, - RandomStringUtils.randomAlphabetic(5), - RandomStringUtils.randomAlphabetic(5)); + return new LineagePojo() + .setParentOddrn(parentOddrn) + .setChildOddrn(RandomStringUtils.randomAlphabetic(5)) + .setEstablisherOddrn(RandomStringUtils.randomAlphabetic(5)); } private LineagePojo generateLineageWithChild(final String childOddrn) { - return new LineagePojo(RandomStringUtils.randomAlphabetic(5), - childOddrn, - RandomStringUtils.randomAlphabetic(5)); + return new LineagePojo() + .setParentOddrn(RandomStringUtils.randomAlphabetic(5)) + .setChildOddrn(childOddrn) + .setEstablisherOddrn(RandomStringUtils.randomAlphabetic(5)); } } \ No newline at end of file diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/DataEntityServiceTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/DataEntityServiceTest.java index e81238fba..a6346b148 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/DataEntityServiceTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/DataEntityServiceTest.java @@ -26,7 +26,6 @@ import org.opendatadiscovery.oddplatform.model.tables.pojos.MetadataFieldPojo; import org.opendatadiscovery.oddplatform.model.tables.pojos.MetadataFieldValuePojo; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataEntityRepository; -import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataEntityStatisticsRepository; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataEntityTaskRunRepository; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDatasetVersionRepository; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveGroupEntityRelationRepository; @@ -35,7 +34,6 @@ import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveMetadataFieldValueRepository; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveSearchEntrypointRepository; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveTagRepository; -import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveTermRepository; import org.opendatadiscovery.oddplatform.service.term.TermService; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -83,8 +81,6 @@ public class DataEntityServiceTest { @Mock private ReactiveGroupEntityRelationRepository reactiveGroupEntityRelationRepository; @Mock - private ReactiveDataEntityStatisticsRepository dataEntityStatisticsRepository; - @Mock private ReactiveTagRepository tagRepository; @Mock private DataEntityFilledService dataEntityFilledService; @@ -93,7 +89,11 @@ public class DataEntityServiceTest { @Mock private DataSourceService dataSourceService; @Mock - private DataEntityInternalInformationService dataEntityInternalInformationService; + private DataEntityInternalStateService dataEntityInternalStateService; + @Mock + private DataEntityStatisticsService dataEntityStatisticsService; + @Mock + private DataEntityRelationsService dataEntityRelationsService; @BeforeEach public void beforeAll() { @@ -103,8 +103,10 @@ public void beforeAll() { dataEntityFilledService, metadataFieldService, dataSourceService, + dataEntityStatisticsService, termService, - dataEntityInternalInformationService, + dataEntityInternalStateService, + dataEntityRelationsService, metadataFieldValueRepository, metadataFieldRepository, reactiveDataEntityRepository, @@ -113,7 +115,6 @@ public void beforeAll() { datasetVersionRepository, reactiveSearchEntrypointRepository, reactiveGroupEntityRelationRepository, - dataEntityStatisticsRepository, tagRepository, dataEntityMapper, metadataFieldMapper, diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/LineageServiceTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/LineageServiceTest.java index a010fd8fd..492923a89 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/LineageServiceTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/LineageServiceTest.java @@ -14,9 +14,11 @@ import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityLineageEdge; import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityLineageNode; import org.opendatadiscovery.oddplatform.dto.DataEntityDimensionsDto; +import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; import org.opendatadiscovery.oddplatform.dto.lineage.LineageStreamKind; import org.opendatadiscovery.oddplatform.mapper.DataEntityMapperImpl; import org.opendatadiscovery.oddplatform.mapper.DataEntityRunMapperImpl; +import org.opendatadiscovery.oddplatform.mapper.DataEntityStatusMapper; import org.opendatadiscovery.oddplatform.mapper.DataSourceMapperImpl; import org.opendatadiscovery.oddplatform.mapper.DatasetFieldApiMapperImpl; import org.opendatadiscovery.oddplatform.mapper.DatasetVersionMapperImpl; @@ -100,7 +102,9 @@ void setUp() { new DateTimeMapperImpl() ), termMapper, - new DateTimeMapperImpl() + new DateTimeMapperImpl(), + new DataEntityStatusMapper(), + new DataEntityStaleDetector() ) ); lineageMapper.setDataSourceMapper( @@ -109,6 +113,8 @@ void setUp() { new TokenMapperImpl(new DateTimeMapperImpl()) ) ); + lineageMapper.setDataEntityStatusMapper(new DataEntityStatusMapper()); + lineageMapper.setDataEntityStaleDetector(new DataEntityStaleDetector()); } @Test @@ -116,12 +122,21 @@ void getLineageTest() { final var rootEntityOddrn = "root"; final var firstChildEntityOddrn = "firstChild"; final var secondChildEntityOddrn = "secondChild"; - final var rootEntity = new DataEntityPojo().setOddrn(rootEntityOddrn).setId(1L) - .setEntityClassIds(new Integer[] {1}); - final var firstChildEntity = new DataEntityPojo().setId(2L).setOddrn(firstChildEntityOddrn); - final var secondChildEntity = new DataEntityPojo().setId(3L).setOddrn(secondChildEntityOddrn); - final var rootToFirstEntityLineage = new LineagePojo(rootEntityOddrn, firstChildEntityOddrn, null); - final var rootToSecondEntityLineage = new LineagePojo(rootEntityOddrn, secondChildEntityOddrn, null); + final var rootEntity = new DataEntityPojo() + .setOddrn(rootEntityOddrn) + .setId(1L) + .setEntityClassIds(new Integer[] {1}) + .setStatus(DataEntityStatusDto.UNASSIGNED.getId()); + final var firstChildEntity = new DataEntityPojo() + .setId(2L) + .setOddrn(firstChildEntityOddrn) + .setStatus(DataEntityStatusDto.UNASSIGNED.getId()); + final var secondChildEntity = new DataEntityPojo() + .setId(3L) + .setOddrn(secondChildEntityOddrn) + .setStatus(DataEntityStatusDto.UNASSIGNED.getId()); + final var rootToFirstEntityLineage = new LineagePojo(rootEntityOddrn, firstChildEntityOddrn, null, false); + final var rootToSecondEntityLineage = new LineagePojo(rootEntityOddrn, secondChildEntityOddrn, null, false); final var dto = DataEntityDimensionsDto.dimensionsBuilder() .dataEntity(rootEntity).build(); diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/NamespaceServiceImplTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/NamespaceServiceImplTest.java index 6eb1895c6..de1a802ed 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/NamespaceServiceImplTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/NamespaceServiceImplTest.java @@ -219,7 +219,7 @@ public void testDelete() { when(dataSourceRepository.existsByNamespace(eq(namespaceId))).thenReturn(Mono.just(false)); when(collectorRepository.existsByNamespace(eq(namespaceId))).thenReturn(Mono.just(false)); when(termRepository.existsByNamespace(eq(namespaceId))).thenReturn(Mono.just(false)); - when(dataEntityRepository.existsByNamespaceId(eq(namespaceId))).thenReturn(Mono.just(false)); + when(dataEntityRepository.existsNonDeletedByNamespaceId(eq(namespaceId))).thenReturn(Mono.just(false)); when(namespaceRepository.delete(eq(namespaceId))).thenReturn(Mono.just(namespace)); namespaceService.delete(namespaceId) @@ -231,7 +231,7 @@ public void testDelete() { verify(dataSourceRepository, only()).existsByNamespace(eq(namespaceId)); verify(collectorRepository, only()).existsByNamespace(eq(namespaceId)); verify(termRepository, only()).existsByNamespace(eq(namespaceId)); - verify(dataEntityRepository, only()).existsByNamespaceId(eq(namespaceId)); + verify(dataEntityRepository, only()).existsNonDeletedByNamespaceId(eq(namespaceId)); } @Test @@ -242,7 +242,7 @@ public void testDeleteTiedNamespaceWithCollector() { when(collectorRepository.existsByNamespace(eq(namespaceId))).thenReturn(Mono.just(true)); when(dataSourceRepository.existsByNamespace(eq(namespaceId))).thenReturn(Mono.just(false)); when(termRepository.existsByNamespace(eq(namespaceId))).thenReturn(Mono.just(false)); - when(dataEntityRepository.existsByNamespaceId(eq(namespaceId))).thenReturn(Mono.just(false)); + when(dataEntityRepository.existsNonDeletedByNamespaceId(eq(namespaceId))).thenReturn(Mono.just(false)); namespaceService.delete(namespaceId) .as(StepVerifier::create) @@ -252,7 +252,7 @@ public void testDeleteTiedNamespaceWithCollector() { verify(dataSourceRepository, only()).existsByNamespace(eq(namespaceId)); verify(collectorRepository, only()).existsByNamespace(eq(namespaceId)); verify(termRepository, only()).existsByNamespace(eq(namespaceId)); - verify(dataEntityRepository, only()).existsByNamespaceId(eq(namespaceId)); + verify(dataEntityRepository, only()).existsNonDeletedByNamespaceId(eq(namespaceId)); } @Test @@ -263,7 +263,7 @@ public void testDeleteTiedNamespaceWithDataSource() { when(collectorRepository.existsByNamespace(eq(namespaceId))).thenReturn(Mono.just(false)); when(dataSourceRepository.existsByNamespace(eq(namespaceId))).thenReturn(Mono.just(true)); when(termRepository.existsByNamespace(eq(namespaceId))).thenReturn(Mono.just(false)); - when(dataEntityRepository.existsByNamespaceId(eq(namespaceId))).thenReturn(Mono.just(false)); + when(dataEntityRepository.existsNonDeletedByNamespaceId(eq(namespaceId))).thenReturn(Mono.just(false)); namespaceService.delete(namespaceId) .as(StepVerifier::create) @@ -272,7 +272,7 @@ public void testDeleteTiedNamespaceWithDataSource() { verify(namespaceRepository, never()).delete(eq(namespaceId)); verify(dataSourceRepository, only()).existsByNamespace(eq(namespaceId)); verify(termRepository, only()).existsByNamespace(eq(namespaceId)); - verify(dataEntityRepository, only()).existsByNamespaceId(eq(namespaceId)); + verify(dataEntityRepository, only()).existsNonDeletedByNamespaceId(eq(namespaceId)); verify(collectorRepository, only()).existsByNamespace(eq(namespaceId)); } @@ -283,7 +283,7 @@ public void testDeleteTiedNamespaceWithTerm() { when(collectorRepository.existsByNamespace(eq(namespaceId))).thenReturn(Mono.just(false)); when(dataSourceRepository.existsByNamespace(eq(namespaceId))).thenReturn(Mono.just(false)); - when(dataEntityRepository.existsByNamespaceId(eq(namespaceId))).thenReturn(Mono.just(false)); + when(dataEntityRepository.existsNonDeletedByNamespaceId(eq(namespaceId))).thenReturn(Mono.just(false)); when(termRepository.existsByNamespace(eq(namespaceId))).thenReturn(Mono.just(true)); namespaceService.delete(namespaceId) @@ -294,7 +294,7 @@ public void testDeleteTiedNamespaceWithTerm() { verify(dataSourceRepository, only()).existsByNamespace(eq(namespaceId)); verify(termRepository, only()).existsByNamespace(eq(namespaceId)); verify(collectorRepository, only()).existsByNamespace(eq(namespaceId)); - verify(dataEntityRepository, only()).existsByNamespaceId(eq(namespaceId)); + verify(dataEntityRepository, only()).existsNonDeletedByNamespaceId(eq(namespaceId)); } @Test @@ -305,7 +305,7 @@ public void testDeleteTiedNamespaceWithDataEntity() { when(collectorRepository.existsByNamespace(eq(namespaceId))).thenReturn(Mono.just(false)); when(dataSourceRepository.existsByNamespace(eq(namespaceId))).thenReturn(Mono.just(false)); when(termRepository.existsByNamespace(eq(namespaceId))).thenReturn(Mono.just(false)); - when(dataEntityRepository.existsByNamespaceId(eq(namespaceId))).thenReturn(Mono.just(true)); + when(dataEntityRepository.existsNonDeletedByNamespaceId(eq(namespaceId))).thenReturn(Mono.just(true)); namespaceService.delete(namespaceId) .as(StepVerifier::create) @@ -315,6 +315,6 @@ public void testDeleteTiedNamespaceWithDataEntity() { verify(dataSourceRepository, only()).existsByNamespace(eq(namespaceId)); verify(termRepository, only()).existsByNamespace(eq(namespaceId)); verify(collectorRepository, only()).existsByNamespace(eq(namespaceId)); - verify(dataEntityRepository, only()).existsByNamespaceId(eq(namespaceId)); + verify(dataEntityRepository, only()).existsNonDeletedByNamespaceId(eq(namespaceId)); } } \ No newline at end of file diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/integration/DataEntityDomainsTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/integration/DataEntityDomainsTest.java new file mode 100644 index 000000000..88f9f2ced --- /dev/null +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/integration/DataEntityDomainsTest.java @@ -0,0 +1,154 @@ +package org.opendatadiscovery.oddplatform.service.integration; + +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.opendatadiscovery.oddplatform.BaseIngestionTest; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityDomain; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityDomainList; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityGroupFormData; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRef; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatus; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusEnum; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityStatusFormData; +import org.opendatadiscovery.oddplatform.api.contract.model.DataSource; +import org.opendatadiscovery.oddplatform.api.ingestion.utils.IngestionModelGenerator; +import org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto; +import org.opendatadiscovery.oddplatform.ingestion.contract.model.DataEntity; +import org.opendatadiscovery.oddplatform.ingestion.contract.model.DataEntityList; +import org.opendatadiscovery.oddplatform.ingestion.contract.model.DataEntityType; +import org.opendatadiscovery.oddplatform.ingestion.contract.model.DataSet; +import org.opendatadiscovery.oddplatform.service.DataEntityService; +import org.springframework.beans.factory.annotation.Autowired; +import reactor.core.publisher.Mono; + +import static org.assertj.core.api.Assertions.assertThat; + +public class DataEntityDomainsTest extends BaseIngestionTest { + /** + * Domains information test. + * + *

We ingest 3 DEG. 2 domains and 1 DAG. Each has 2 children. Assert correct information + * + *

We delete 1 domain and 1 children from the second domain. Assert again + */ + @Test + void testDomainsInfo() { + final DataSource createdDataSource = createDataSource(); + final DataEntity firstChild = IngestionModelGenerator.generateSimpleDataEntity(DataEntityType.TABLE) + .dataset(new DataSet().rowsNumber(10L).fieldList(IngestionModelGenerator.generateDatasetFields(5))); + final DataEntity secondChild = IngestionModelGenerator.generateSimpleDataEntity(DataEntityType.TABLE) + .dataset(new DataSet().rowsNumber(10L).fieldList(IngestionModelGenerator.generateDatasetFields(5))); + final DataEntity thirdChild = IngestionModelGenerator.generateSimpleDataEntity(DataEntityType.TABLE) + .dataset(new DataSet().rowsNumber(10L).fieldList(IngestionModelGenerator.generateDatasetFields(5))); + final DataEntity fourthChild = IngestionModelGenerator.generateSimpleDataEntity(DataEntityType.TABLE) + .dataset(new DataSet().rowsNumber(10L).fieldList(IngestionModelGenerator.generateDatasetFields(5))); + final DataEntity fifthChild = IngestionModelGenerator.generateSimpleDataEntity(DataEntityType.TABLE) + .dataset(new DataSet().rowsNumber(10L).fieldList(IngestionModelGenerator.generateDatasetFields(5))); + final DataEntity sixthChild = IngestionModelGenerator.generateSimpleDataEntity(DataEntityType.TABLE) + .dataset(new DataSet().rowsNumber(10L).fieldList(IngestionModelGenerator.generateDatasetFields(5))); + + final List items = + List.of(firstChild, secondChild, thirdChild, fourthChild, fifthChild, sixthChild); + + final var dataEntityList = new DataEntityList() + .dataSourceOddrn(createdDataSource.getOddrn()) + .items(items); + ingestAndAssert(dataEntityList); + + final Map ingestedEntities = extractIngestedEntitiesAndAssert(createdDataSource, items.size()); + + final DataEntityGroupFormData firstDomain = + createDEGFormData(DataEntityTypeDto.DOMAIN, List.of(firstChild, secondChild), ingestedEntities); + final DataEntityGroupFormData secondDomain = + createDEGFormData(DataEntityTypeDto.DOMAIN, List.of(thirdChild, fourthChild), ingestedEntities); + final DataEntityGroupFormData dag = + createDEGFormData(DataEntityTypeDto.DAG, List.of(fifthChild, sixthChild), ingestedEntities); + + final DataEntityRef firstDomainRef = createDEG(firstDomain); + final DataEntityRef secondDomainRef = createDEG(secondDomain); + createDEG(dag); + + final DataEntityDomainList domainsInfo = getDomainsInfo(); + assertThat(domainsInfo.getItems()).hasSize(2); + assertThat(domainsInfo.getItems()).hasSameElementsAs(List.of( + new DataEntityDomain(firstDomainRef, 2L), + new DataEntityDomain(secondDomainRef, 2L) + )); + + unAssignEntityFromDeg(ingestedEntities.get(firstChild.getOddrn()), firstDomainRef.getId()); + unAssignEntityFromDeg(ingestedEntities.get(secondChild.getOddrn()), firstDomainRef.getId()); + changeStatus(firstDomainRef.getId(), new DataEntityStatusFormData() + .status(new DataEntityStatus(DataEntityStatusEnum.DELETED))); + + changeStatus(ingestedEntities.get(thirdChild.getOddrn()), + new DataEntityStatusFormData().status(new DataEntityStatus(DataEntityStatusEnum.DELETED))); + + final DataEntityDomainList updatedDomainsInfo = getDomainsInfo(); + assertThat(updatedDomainsInfo.getItems()).hasSize(1); + assertThat(updatedDomainsInfo.getItems()).hasSameElementsAs(List.of( + new DataEntityDomain(secondDomainRef, 1L) + )); + } + + private DataEntityDomainList getDomainsInfo() { + return webTestClient.get() + .uri("/api/dataentitygroups/domains") + .exchange() + .returnResult(DataEntityDomainList.class) + .getResponseBody() + .single() + .block(); + } + + private DataEntityRef createDEG(final DataEntityGroupFormData formData) { + return webTestClient.post() + .uri("/api/dataentitygroups") + .body(Mono.just(formData), DataEntityGroupFormData.class) + .exchange() + .returnResult(DataEntityRef.class) + .getResponseBody() + .single() + .block(); + } + + private void unAssignEntityFromDeg(final Long dataEntityId, + final Long dataEntityGroupId) { + webTestClient.delete() + .uri("/api/dataentities/{data_entity_id}/data_entity_group/{data_entity_group_id}", + dataEntityId, + dataEntityGroupId) + .exchange() + .expectStatus().isNoContent(); + } + + private void changeStatus(final Long dataEntityId, + final DataEntityStatusFormData status) { + webTestClient.put() + .uri("/api/dataentities/{data_entity_id}/statuses", dataEntityId) + .body(Mono.just(status), DataEntityStatusFormData.class) + .exchange() + .expectStatus().isOk(); + } + + private DataEntityGroupFormData createDEGFormData(final DataEntityTypeDto type, + final List children, + final Map ingestedMap) { + final DataEntityGroupFormData formData = new DataEntityGroupFormData(); + formData.setName(UUID.randomUUID().toString()); + formData.setType(new org.opendatadiscovery.oddplatform.api.contract.model.DataEntityType() + .id(type.getId()) + .name(org.opendatadiscovery.oddplatform.api.contract.model.DataEntityType.NameEnum.fromValue( + type.name()))); + final List dataEntityRefs = children.stream() + .map(de -> new DataEntityRef() + .id(ingestedMap.get(de.getOddrn())) + .oddrn(de.getOddrn()) + .status(new DataEntityStatus(DataEntityStatusEnum.UNASSIGNED)) + .isStale(false)) + .toList(); + formData.setEntities(dataEntityRefs); + return formData; + } +} diff --git a/odd-platform-api/src/test/resources/policy/valid/combined_policy.json b/odd-platform-api/src/test/resources/policy/valid/combined_policy.json index e5ae3dde5..f1cf87902 100644 --- a/odd-platform-api/src/test/resources/policy/valid/combined_policy.json +++ b/odd-platform-api/src/test/resources/policy/valid/combined_policy.json @@ -70,8 +70,7 @@ "DATASET_FIELD_DESCRIPTION_UPDATE", "DATASET_FIELD_LABELS_UPDATE", "DATASET_FIELD_ENUMS_UPDATE", - "DATA_ENTITY_GROUP_UPDATE", - "DATA_ENTITY_GROUP_DELETE" + "DATA_ENTITY_GROUP_UPDATE" ] }, { diff --git a/odd-platform-specification/components.yaml b/odd-platform-specification/components.yaml index 3ef514138..aeea1a778 100644 --- a/odd-platform-specification/components.yaml +++ b/odd-platform-specification/components.yaml @@ -181,8 +181,8 @@ components: - DATASET_FIELD_DELETE_TERM - DATA_ENTITY_GROUP_CREATE - DATA_ENTITY_GROUP_UPDATE - - DATA_ENTITY_GROUP_DELETE - DATA_ENTITY_ATTACHMENT_MANAGE + - DATA_ENTITY_STATUS_UPDATE - TERM_CREATE - TERM_UPDATE - TERM_DELETE @@ -541,15 +541,22 @@ components: $ref: '#/components/schemas/DataEntityType' data_entity_groups: $ref: '#/components/schemas/DataEntityRefList' - created_at: + status: + $ref: '#/components/schemas/DataEntityStatus' + source_created_at: type: string format: date-time - updated_at: + source_updated_at: + type: string + format: date-time + last_ingested_at: type: string format: date-time view_count: type: integer format: int64 + is_stale: + type: boolean required: - id - oddrn @@ -557,7 +564,9 @@ components: - data_source - type - namespace + - status - view_count + - is_stale DataEntityDetailsBaseObject: type: object @@ -734,6 +743,26 @@ components: - id - name + DataEntityStatusEnum: + type: string + enum: + - UNASSIGNED + - DRAFT + - STABLE + - DEPRECATED + - DELETED + + DataEntityStatus: + type: object + properties: + status: + $ref: '#/components/schemas/DataEntityStatusEnum' + status_switch_time: + type: string + format: date-time + required: + - status + DataSetStats: type: object properties: @@ -828,8 +857,14 @@ components: type: boolean manually_created: type: boolean + status: + $ref: '#/components/schemas/DataEntityStatus' + is_stale: + type: boolean required: - id + - status + - is_stale DataEntityRefList: type: array @@ -1032,6 +1067,14 @@ components: - is_upper_group - data_entity + DataEntityStatusFormData: + type: object + properties: + status: + $ref: '#/components/schemas/DataEntityStatus' + propagate: + type: boolean + DataSetSLAReport: type: object properties: @@ -1290,6 +1333,7 @@ components: - tags - types - groups + - statuses SearchFilter: type: object @@ -1347,6 +1391,10 @@ components: type: array items: $ref: '#/components/schemas/SearchFilter' + statuses: + type: array + items: + $ref: '#/components/schemas/SearchFilter' required: - entity_classes @@ -1848,11 +1896,17 @@ components: type: integer children_count: type: integer + status: + $ref: '#/components/schemas/DataEntityStatus' + is_stale: + type: boolean required: - id - oddrn - name - type + - status + - is_stale DataEntityLineageEdge: type: object @@ -2116,6 +2170,10 @@ components: type: array items: $ref: '#/components/schemas/SearchFilterState' + statuses: + type: array + items: + $ref: '#/components/schemas/SearchFilterState' required: - filters @@ -2572,6 +2630,8 @@ components: $ref: '#/components/schemas/DescriptionActivityState' business_name: $ref: '#/components/schemas/BusinessNameActivityState' + status: + $ref: '#/components/schemas/DataEntityStatusActivityState' custom_metadata: type: array items: @@ -2656,6 +2716,12 @@ components: internal_name: type: string + DataEntityStatusActivityState: + type: object + properties: + status: + $ref: '#/components/schemas/DataEntityStatus' + CustomMetadataActivityState: type: object properties: @@ -2841,6 +2907,7 @@ components: - TERM_ASSIGNMENT_UPDATED - DESCRIPTION_UPDATED - BUSINESS_NAME_UPDATED + - DATA_ENTITY_STATUS_UPDATED - CUSTOM_METADATA_CREATED - CUSTOM_METADATA_UPDATED - CUSTOM_METADATA_DELETED @@ -2850,7 +2917,6 @@ components: - DATASET_FIELD_TERM_ASSIGNMENT_UPDATED - CUSTOM_GROUP_CREATED - CUSTOM_GROUP_UPDATED - - CUSTOM_GROUP_DELETED - ALERT_HALT_CONFIG_UPDATED - ALERT_STATUS_UPDATED - OPEN_ALERT_RECEIVED diff --git a/odd-platform-specification/openapi.yaml b/odd-platform-specification/openapi.yaml index 89db2f044..06a215394 100644 --- a/odd-platform-specification/openapi.yaml +++ b/odd-platform-specification/openapi.yaml @@ -1071,6 +1071,29 @@ paths: tags: - dataEntity + /api/dataentities/{data_entity_id}/statuses: + put: + summary: Update status to data entity + description: Update deprecation status to data entity + operationId: updateStatus + parameters: + - $ref: './components.yaml/#/components/parameters/DataEntityIdParam' + requestBody: + required: true + content: + application/json: + schema: + $ref: './components.yaml/#/components/schemas/DataEntityStatusFormData' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: './components.yaml/#/components/schemas/DataEntityStatus' + tags: + - dataEntity + /api/dataentities/{data_entity_id}/terms/{term_id}: delete: summary: Delete term from Data entity @@ -2067,17 +2090,6 @@ paths: $ref: './components.yaml/#/components/schemas/DataEntityRef' tags: - dataEntity - delete: - summary: Delete DataEntityGroup - description: Deletes DataEntityGroup - operationId: deleteDataEntityGroup - parameters: - - $ref: './components.yaml/#/components/parameters/DataEntityGroupIdParam' - responses: - '204': - $ref: './components.yaml/#/components/responses/Deleted' - tags: - - dataEntity /api/dataentitygroups/{data_entity_group_id}/lineage: get: diff --git a/odd-platform-ui/src/components/Activity/ActivityResults/ActivityItem/ActivityItem.tsx b/odd-platform-ui/src/components/Activity/ActivityResults/ActivityItem/ActivityItem.tsx index c21821e43..12b4dc3a1 100644 --- a/odd-platform-ui/src/components/Activity/ActivityResults/ActivityItem/ActivityItem.tsx +++ b/odd-platform-ui/src/components/Activity/ActivityResults/ActivityItem/ActivityItem.tsx @@ -13,10 +13,11 @@ import { OwnerActivityField, StringActivityField, TermActivityField, + DatasetTermActivityField, + EntityStatusActivityField, } from 'components/shared/elements/Activity'; import { useAppDateTime, useAppPaths } from 'lib/hooks'; import type { Activity } from 'redux/interfaces'; -import DatasetTermActivityField from 'components/shared/elements/Activity/ActivityFields/DatasetTermActivityField/DatasetTermActivityField'; import * as S from './ActivityItemStyles'; interface ActivityItemProps { @@ -182,13 +183,6 @@ const ActivityItem: React.FC = ({ activityName={`${activity.dataEntity.internalName}`} /> )} - {isTypeRelatedTo([ActivityEventType.CUSTOM_GROUP_DELETED]) && ( - - )} {isTypeRelatedTo([ActivityEventType.CUSTOM_GROUP_UPDATED]) && ( = ({ newState={activity.newState} /> )} + {isTypeRelatedTo([ActivityEventType.DATA_ENTITY_STATUS_UPDATED]) && ( + + )} ); }; diff --git a/odd-platform-ui/src/components/DataEntityDetails/DataEntityActivity/ActivityResults/ActivityItem/ActivityItem.tsx b/odd-platform-ui/src/components/DataEntityDetails/DataEntityActivity/ActivityResults/ActivityItem/ActivityItem.tsx index af456dca4..469c62325 100644 --- a/odd-platform-ui/src/components/DataEntityDetails/DataEntityActivity/ActivityResults/ActivityItem/ActivityItem.tsx +++ b/odd-platform-ui/src/components/DataEntityDetails/DataEntityActivity/ActivityResults/ActivityItem/ActivityItem.tsx @@ -13,6 +13,7 @@ import { StringActivityField, TermActivityField, DatasetTermActivityField, + EntityStatusActivityField, } from 'components/shared/elements/Activity'; import { useAppDateTime } from 'lib/hooks'; import { type ActivityItemProps } from 'components/shared/elements/Activity/common'; @@ -134,13 +135,6 @@ const ActivityItem: React.FC = ({ activity, hideAllDetails }) activityName={`${activity.dataEntity.internalName}`} /> )} - {isTypeRelatedTo([ActivityEventType.CUSTOM_GROUP_DELETED]) && ( - - )} {isTypeRelatedTo([ActivityEventType.CUSTOM_GROUP_UPDATED]) && ( = ({ activity, hideAllDetails }) newState={activity.newState} /> )} + {isTypeRelatedTo([ActivityEventType.DATA_ENTITY_STATUS_UPDATED]) && ( + + )} {activity.systemEvent ? ( diff --git a/odd-platform-ui/src/components/DataEntityDetails/DataEntityDetails.tsx b/odd-platform-ui/src/components/DataEntityDetails/DataEntityDetails.tsx index a876be4c8..217142ea7 100644 --- a/odd-platform-ui/src/components/DataEntityDetails/DataEntityDetails.tsx +++ b/odd-platform-ui/src/components/DataEntityDetails/DataEntityDetails.tsx @@ -1,5 +1,5 @@ import { Grid } from '@mui/material'; -import React from 'react'; +import React, { useEffect } from 'react'; import { AppErrorPage, SkeletonWrapper } from 'components/shared/elements'; import { useAppParams } from 'lib/hooks'; import { @@ -53,16 +53,17 @@ const DataEntityDetails: React.FC = () => { getDataEntityDeleteFromGroupStatuses ); - React.useEffect(() => { + useEffect(() => { dispatch(fetchDataEntityDetails({ dataEntityId })); }, [ dataEntityId, isDataEntityGroupUpdated, isDataEntityAddedToGroup, isDataEntityDeletedFromGroup, + details.status?.status, ]); - React.useEffect(() => { + useEffect(() => { dispatch(fetchDataEntityAlertsCounts({ dataEntityId, status: AlertStatus.OPEN })); dispatch(fetchDataSetQualityTestReport({ dataEntityId })); dispatch(fetchDataSetQualitySLAReport({ dataEntityId })); @@ -82,7 +83,7 @@ const DataEntityDetails: React.FC = () => { allowedPermissions={[ Permission.DATA_ENTITY_INTERNAL_NAME_UPDATE, Permission.DATA_ENTITY_GROUP_UPDATE, - Permission.DATA_ENTITY_GROUP_DELETE, + Permission.DATA_ENTITY_STATUS_UPDATE, ]} resourcePermissions={resourcePermissions} render={() => ( @@ -93,7 +94,9 @@ const DataEntityDetails: React.FC = () => { entityClasses={details.entityClasses} type={details.type} manuallyCreated={details.manuallyCreated} - updatedAt={details.updatedAt} + lastIngestedAt={details.lastIngestedAt} + isStale={details.isStale} + status={details.status} /> )} /> diff --git a/odd-platform-ui/src/components/DataEntityDetails/DataEntityDetailsHeader/DataEntityDetailsHeader.tsx b/odd-platform-ui/src/components/DataEntityDetails/DataEntityDetailsHeader/DataEntityDetailsHeader.tsx index 77c480820..97d892a62 100644 --- a/odd-platform-ui/src/components/DataEntityDetails/DataEntityDetailsHeader/DataEntityDetailsHeader.tsx +++ b/odd-platform-ui/src/components/DataEntityDetails/DataEntityDetailsHeader/DataEntityDetailsHeader.tsx @@ -3,17 +3,21 @@ import { Grid, Typography } from '@mui/material'; import { Button, EntityClassItem, + EntityStatus, EntityTypeItem, LabelItem, + MetadataStale, WithFeature, } from 'components/shared/elements'; import { WithPermissions } from 'components/shared/contexts'; import { type DataEntityDetails, Feature, Permission } from 'generated-sources'; import { AddIcon, EditIcon, SlackIcon, TimeGapIcon } from 'components/shared/icons'; import { useAppDateTime } from 'lib/hooks'; +import { useAppSelector } from 'redux/lib/hooks'; +import { getIsDataEntityBelongsToClass, getIsEntityStatusDeleted } from 'redux/selectors'; +import DataEntityGroupForm from '../DataEntityGroup/DataEntityGroupForm/DataEntityGroupForm'; import CreateMessageForm from '../DataCollaboration/CreateMessageForm/CreateMessageForm'; import InternalNameFormDialog from '../InternalNameFormDialog/InternalNameFormDialog'; -import DataEntityGroupControls from '../DataEntityGroup/DataEntityGroupControls/DataEntityGroupControls'; interface DataEntityDetailsHeaderProps { dataEntityId: DataEntityDetails['id']; @@ -22,50 +26,51 @@ interface DataEntityDetailsHeaderProps { entityClasses: DataEntityDetails['entityClasses']; type: DataEntityDetails['type']; manuallyCreated: DataEntityDetails['manuallyCreated']; - updatedAt: DataEntityDetails['updatedAt']; + lastIngestedAt: DataEntityDetails['lastIngestedAt']; + status: DataEntityDetails['status']; + isStale: DataEntityDetails['isStale']; } const DataEntityDetailsHeader: React.FC = ({ - updatedAt, + lastIngestedAt, entityClasses, manuallyCreated, externalName, internalName, type, dataEntityId, + status, + isStale, }) => { const { formatDistanceToNowStrict } = useAppDateTime(); + const { isDEG } = useAppSelector(getIsDataEntityBelongsToClass(dataEntityId)); + const isStatusDeleted = useAppSelector(getIsEntityStatusDeleted(dataEntityId)); - const entityUpdatedAt = React.useMemo( - () => - updatedAt && ( - <> - - - {formatDistanceToNowStrict(updatedAt, { addSuffix: true })} - - - ), - [updatedAt] - ); + const entityLastIngestedAt = lastIngestedAt ? ( + <> + {isStale ? ( + + ) : ( + + )} + + {formatDistanceToNowStrict(lastIngestedAt, { addSuffix: true })} + + + ) : null; - const originalName = React.useMemo( - () => - internalName && - externalName && ( - - - - {externalName} - - - ), - [internalName, externalName] + const originalName = internalName && externalName && ( + + + + {externalName} + + ); return ( - + {internalName || externalName} @@ -77,35 +82,50 @@ const DataEntityDetailsHeader: React.FC = ({ /> ))} {type && } - - : } - /> - } - /> - + {!isStatusDeleted && ( + + : } + /> + } + /> + + )} - {entityUpdatedAt} - {manuallyCreated && ( - + {entityLastIngestedAt} + ( + + )} + /> + {manuallyCreated && !isStatusDeleted && ( + + + } + /> + )} import('../Overview/Overview')); const DatasetStructure = React.lazy(() => import('../DatasetStructure/DatasetStructure')); @@ -33,6 +33,7 @@ const DataEntityDetailsRoutes: React.FC = () => { const resourcePermissions = useAppSelector( getResourcePermissions(PermissionResourceType.DATA_ENTITY, dataEntityId) ); + const isStatusDeleted = useAppSelector(getIsEntityStatusDeleted(dataEntityId)); return ( @@ -47,10 +48,25 @@ const DataEntityDetailsRoutes: React.FC = () => { - } /> + + } + /> } + element={ + + } > { + + + } + /> + + } + /> + } /> - } /> - } /> } /> } + element={ + + } > diff --git a/odd-platform-ui/src/components/DataEntityDetails/DataEntityDetailsTabs/DataEntityDetailsTabs.tsx b/odd-platform-ui/src/components/DataEntityDetails/DataEntityDetailsTabs/DataEntityDetailsTabs.tsx index 79c4e9ce6..3041e1446 100644 --- a/odd-platform-ui/src/components/DataEntityDetails/DataEntityDetailsTabs/DataEntityDetailsTabs.tsx +++ b/odd-platform-ui/src/components/DataEntityDetails/DataEntityDetailsTabs/DataEntityDetailsTabs.tsx @@ -6,6 +6,7 @@ import { getDataEntityDetails, getDatasetTestReportTotal, getIsDataEntityBelongsToClass, + getIsEntityStatusDeleted, } from 'redux/selectors'; import { useAppSelector } from 'redux/lib/hooks'; import { @@ -43,6 +44,7 @@ const DataEntityDetailsTabs: React.FC = () => { const { isDataset, isQualityTest, isTransformer, isDEG } = useAppSelector( getIsDataEntityBelongsToClass(dataEntityId) ); + const isStatusDeleted = useAppSelector(getIsEntityStatusDeleted(dataEntityId)); const tabs = React.useMemo( () => [ @@ -63,19 +65,19 @@ const DataEntityDetailsTabs: React.FC = () => { dataEntityId, isDEG ? degLineageQueryString : lineageQueryString ), - hidden: isQualityTest, + hidden: isQualityTest || isStatusDeleted, value: DataEntityRoutes.lineage, }, { name: 'Test reports', link: dataEntityTestReportPath(dataEntityId), - hidden: !isDataset || !datasetQualityTestReportTotal, + hidden: !isDataset || !datasetQualityTestReportTotal || isStatusDeleted, value: DataEntityRoutes.testReports, }, { name: 'History', link: dataEntityHistoryPath(dataEntityId), - hidden: !isQualityTest && !isTransformer, + hidden: (!isQualityTest && !isTransformer) || isStatusDeleted, value: DataEntityRoutes.history, }, { @@ -84,11 +86,12 @@ const DataEntityDetailsTabs: React.FC = () => { value: DataEntityRoutes.alerts, hint: openAlertsCount > 0 ? openAlertsCount : undefined, hintType: 'alert', + hidden: isStatusDeleted, }, { name: 'Linked items', link: dataEntityLinkedItemsPath(dataEntityId), - hidden: !dataEntityDetails?.hasChildren, + hidden: !dataEntityDetails?.hasChildren || isStatusDeleted, value: DataEntityRoutes.linkedItems, }, { @@ -100,12 +103,13 @@ const DataEntityDetailsTabs: React.FC = () => { name: 'Discussions', link: dataEntityCollaborationPath(dataEntityId), value: DataEntityRoutes.discussions, + hidden: isStatusDeleted, }, ], [ dataEntityId, activityQueryString, - dataEntityDetails, + dataEntityDetails?.hasChildren, openAlertsCount, isDataset, isQualityTest, @@ -114,6 +118,8 @@ const DataEntityDetailsTabs: React.FC = () => { isDEG, degLineageQueryString, lineageQueryString, + dataEntityDetails.status, + isStatusDeleted, ] ); diff --git a/odd-platform-ui/src/components/DataEntityDetails/DataEntityGroup/DataEntityGroupControls/DataEntityGroupControls.tsx b/odd-platform-ui/src/components/DataEntityDetails/DataEntityGroup/DataEntityGroupControls/DataEntityGroupControls.tsx deleted file mode 100644 index a87e861ec..000000000 --- a/odd-platform-ui/src/components/DataEntityDetails/DataEntityGroup/DataEntityGroupControls/DataEntityGroupControls.tsx +++ /dev/null @@ -1,91 +0,0 @@ -import { Grid } from '@mui/material'; -import React from 'react'; -import { useNavigate } from 'react-router-dom'; -import { - AppMenuItem, - AppPopover, - Button, - ConfirmationDialog, -} from 'components/shared/elements'; -import { KebabIcon } from 'components/shared/icons'; -import { useAppParams, useAppPaths } from 'lib/hooks'; -import { deleteDataEntityGroup } from 'redux/thunks'; -import { getSearchId } from 'redux/selectors'; -import { Permission } from 'generated-sources'; -import { useAppDispatch, useAppSelector } from 'redux/lib/hooks'; -import { WithPermissions } from 'components/shared/contexts'; -import DataEntityGroupForm from '../DataEntityGroupForm/DataEntityGroupForm'; - -interface DataEntityGroupControlsProps { - internalName: string | undefined; - externalName: string | undefined; -} - -const DataEntityGroupControls: React.FC = ({ - internalName, - externalName, -}) => { - const dispatch = useAppDispatch(); - const navigate = useNavigate(); - const { dataEntityId } = useAppParams(); - const { searchPath } = useAppPaths(); - - const searchId = useAppSelector(getSearchId); - - const handleEntityGroupDelete = React.useCallback( - () => - dispatch(deleteDataEntityGroup({ dataEntityGroupId: dataEntityId })).then(() => { - navigate(searchPath(searchId)); - }), - [deleteDataEntityGroup, dataEntityId] - ); - - return ( - - ( -