Skip to content

Harvesters / Check if the metadata schema of a metadata to update has changed #9052

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Harvesters / Check if the metadata schema of a metadata to update has changed #9052

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//=============================================================================
//=== Copyright (C) 2001-2024 Food and Agriculture Organization of the
//=== Copyright (C) 2001-2025 Food and Agriculture Organization of the
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
//=== and United Nations Environment Programme (UNEP)
//===
Expand Down Expand Up @@ -66,8 +66,6 @@
import javax.transaction.Transactional;
import javax.transaction.Transactional.TxType;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;

Expand Down Expand Up @@ -309,7 +307,7 @@ private void addMetadata(RecordInfo ri, String uuidToAssign) throws Exception {
// If the xslfilter process changes the metadata uuid,
// use that uuid (newMdUuid) for the new metadata to add to the catalogue.
String newMdUuid = null;
if (!params.xslfilter.equals("")) {
if (!params.xslfilter.isEmpty()) {
md = applyXSLTProcessToMetadata(context, md, processName, processParams, log);
schema = dataMan.autodetectSchema(md);
// Get new uuid if modified by XSLT process
Expand Down Expand Up @@ -467,6 +465,11 @@ boolean updatingLocalMetadata(RecordInfo ri, String id, boolean force) throws Ex
String newSchema = dataMan.autodetectSchema(md);
updateSchema = !newSchema.equals(schema);
schema = newSchema;
} else {
if (!ri.schema.equals(schema)) {
log.warning(" - Detected schema '" + schema + "' is different from the one of the metadata in the catalog '" + ri.schema + "'. Using the detected one.");
updateSchema = true;
}
}

applyBatchEdits(ri.uuid, md, schema, params.getBatchEdits(), context, log);
Expand Down
12 changes: 9 additions & 3 deletions .../main/java/org/fao/geonet/kernel/harvest/harvester/database/DatabaseHarvesterAligner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ private String processMetadata(Element metadataElement) throws Exception {

switch (params.getOverrideUuid()) {
case OVERRIDE:
updateMetadata(metadataElement, Integer.toString(metadataUtils.findOneByUuid(uuid).getId()), true);
updateMetadata(metadataElement, metadataUtils.findOneByUuid(uuid), true);
log.debug(String.format("Overriding record with uuid %s", uuid));
result.updatedMetadata++;
break;
Expand All @@ -253,14 +253,15 @@ private String processMetadata(Element metadataElement) throws Exception {
}
} else {
//record exists and belongs to this harvester
updateMetadata(metadataElement, id, false);
updateMetadata(metadataElement, metadataUtils.findOne(id), false);
result.updatedMetadata++;
}

return id;
}

private void updateMetadata(Element xml, String id, boolean force) throws Exception {
private void updateMetadata(Element xml, AbstractMetadata originalMetadata, boolean force) throws Exception {
String id = Integer.toString(originalMetadata.getId());
log.info("Updating metadata with id: " + id);

//
Expand Down Expand Up @@ -288,6 +289,11 @@ private void updateMetadata(Element xml, String id, boolean force) throws Except
String newSchema = metadataSchemaUtils.autodetectSchema(xml);
updateSchema = (newSchema != null) && !newSchema.equals(schema);
schema = newSchema;
} else {
if (!originalMetadata.getDataInfo().getSchemaId().equals(schema)) {
log.warning(" - Detected schema '" + schema + "' is different from the one of the metadata in the catalog '" + originalMetadata.getDataInfo().getSchemaId() + "'. Using the detected one.");
updateSchema = true;
}
}

applyBatchEdits(uuid, xml, schema, params.getBatchEdits(), context, log);
Expand Down
26 changes: 22 additions & 4 deletions ...s/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/BaseGeoNetworkAligner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -432,10 +432,22 @@ private void updateMetadata(RecordInfo ri, String id, Element md,
MetadataResourceDatabaseMigration.updateMetadataResourcesLink(md, null, settingManager);
}

String schema = dataMan.autodetectSchema(md, null);
boolean updateSchema = false;

if (!params.xslfilter.isEmpty()) {
md = HarvesterUtil.processMetadata(metadataSchemaUtils.getSchema(ri.schema),
md, processName, processParams);
String newSchema = dataMan.autodetectSchema(md);
updateSchema = !newSchema.equals(schema);
schema = newSchema;
} else {
if (!ri.schema.equals(schema)) {
log.warning(" - Detected schema '" + schema + "' is different from the one of the metadata in the catalog '" + ri.schema + "'. Using the detected one.");
updateSchema = true;
}
}

// update metadata
if (log.isDebugEnabled()) {
log.debug(" - Updating local metadata with id=" + id);
Expand All @@ -449,10 +461,16 @@ private void updateMetadata(RecordInfo ri, String id, Element md,
updateDateStamp, IndexingMode.none);
metadata = metadataRepository.findOneById(Integer.parseInt(id));
result.updatedMetadata++;
if (force) {
//change ownership of metadata to new harvester
metadata.getHarvestInfo().setUuid(params.getUuid());
metadata.getSourceInfo().setSourceId(params.getUuid());
if (force || updateSchema) {
if (force) {
//change ownership of metadata to new harvester
metadata.getHarvestInfo().setUuid(params.getUuid());
metadata.getSourceInfo().setSourceId(params.getUuid());
}

if (updateSchema) {
metadata.getDataInfo().setSchemaId(schema);
}

metadataManager.save(metadata);
}
Expand Down
24 changes: 11 additions & 13 deletions harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//=============================================================================
//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the
//=== Copyright (C) 2001-2025 Food and Agriculture Organization of the
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
//=== and United Nations Environment Programme (UNEP)
//===
Expand Down Expand Up @@ -28,7 +28,6 @@
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -64,7 +63,6 @@
import org.fao.geonet.lib.Lib;
import org.fao.geonet.repository.MetadataValidationRepository;
import org.fao.geonet.repository.OperationAllowedRepository;
import org.fao.geonet.repository.Updater;
import org.fao.geonet.repository.specification.MetadataValidationSpecs;
import org.fao.geonet.utils.GeonetHttpRequestFactory;
import org.fao.geonet.utils.Xml;
Expand All @@ -81,7 +79,6 @@

import jeeves.server.context.ServiceContext;

import javax.annotation.Nonnull;

//=============================================================================

Expand Down Expand Up @@ -317,7 +314,7 @@ private void align(XmlRequest t, Set<RecordInfo> records) throws Exception {
switch (params.getOverrideUuid()) {
case OVERRIDE:
processParams.put("mdChangeDate", ri.changeDate);
updateMetadata(t, ri, Integer.toString(metadataUtils.findOneByUuid(ri.id).getId()),
updateMetadata(t, ri, metadataUtils.findOneByUuid(ri.id),
processName, processParams, true);
result.updatedMetadata++;
break;
Expand All @@ -340,7 +337,7 @@ private void align(XmlRequest t, Set<RecordInfo> records) throws Exception {
//record exists and belongs to this harvester
String id = localUuids.getID(ri.id);
processParams.put("mdChangeDate", ri.changeDate);
updateMetadata(t, ri, id, processName, processParams, false);
updateMetadata(t, ri, metadataUtils.findOne(id), processName, processParams, false);
}
result.totalMetadata++;
} catch (Throwable tr) {
Expand Down Expand Up @@ -534,7 +531,8 @@ private Element toDublinCore(Element md) {
}
}

private void updateMetadata(XmlRequest t, RecordInfo ri, String id, String processName, Map<String, Object> processParams, boolean force) throws Exception {
private void updateMetadata(XmlRequest t, RecordInfo ri, AbstractMetadata originalMetadata, String processName, Map<String, Object> processParams, boolean force) throws Exception {
String id = Integer.toString(originalMetadata.getId());
String date = localUuids.getChangeDate(ri.id);

if (!force && !ri.isMoreRecentThan(date)) {
Expand Down Expand Up @@ -565,6 +563,11 @@ private void updateMetadata(XmlRequest t, RecordInfo ri, String id, String proce

schema = dataMan.autodetectSchema(md);
updateSchema = true;
} else {
if (!originalMetadata.getDataInfo().getSchemaId().equals(schema)) {
log.warning(" - Detected schema '" + schema + "' is different from the one of the metadata in the catalog '" + originalMetadata.getDataInfo().getSchemaId() + "'. Using the detected one.");
updateSchema = true;
}
}

// Translate metadata
Expand All @@ -585,12 +588,7 @@ private void updateMetadata(XmlRequest t, RecordInfo ri, String id, String proce
context.getBean(MetadataValidationRepository.class);

final String newSchema = schema;
metadataManager.update(Integer.parseInt(id), new Updater<AbstractMetadata>() {
@Override
public void apply(@Nonnull AbstractMetadata entity) {
entity.getDataInfo().setSchemaId(newSchema);
}
});
metadataManager.update(Integer.parseInt(id), entity -> entity.getDataInfo().setSchemaId(newSchema));

metadataValidationRepository.deleteAll(MetadataValidationSpecs.hasMetadataId(Integer.parseInt(id)));
}
Expand Down
5 changes: 5 additions & 0 deletions harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/sftp/Aligner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,11 @@ boolean updatingLocalMetadata(RecordInfo ri, String id, Element md, boolean forc
String newSchema = metadataSchemaUtils.autodetectSchema(md);
updateSchema = !newSchema.equals(schema);
schema = newSchema;
} else {
if (!ri.schema.equals(schema)) {
log.warning(" - Detected schema '" + schema + "' is different from the one of the metadata in the catalog '" + ri.schema + "'. Using the detected one.");
updateSchema = true;
}
}

boolean validate = false;
Expand Down
23 changes: 19 additions & 4 deletions harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -489,9 +489,18 @@ private void updateMetadata(RemoteFile rf, RecordInfo recordInfo, boolean force)
md = translateMetadataContent(context, md, schema);
}

boolean updateSchema = false;
if (StringUtils.isNotEmpty(params.xslfilter)) {
md = HarvesterUtil.processMetadata(dataMan.getSchema(schema),
md, processName, processParams);
String newSchema = dataMan.autodetectSchema(md);
updateSchema = !newSchema.equals(schema);
schema = newSchema;
} else {
if (!recordInfo.schema.equals(schema)) {
log.warning(" - Detected schema '" + schema + "' is different from the one of the metadata in the catalog '" + recordInfo.schema + "'. Using the detected one.");
updateSchema = true;
}
}

//
Expand All @@ -504,10 +513,16 @@ private void updateMetadata(RemoteFile rf, RecordInfo recordInfo, boolean force)
final AbstractMetadata metadata = metadataManager.updateMetadata(context, recordInfo.id, md, validate, ufo, language,
date, true, IndexingMode.none);

if(force) {
//change ownership of metadata to new harvester
metadata.getHarvestInfo().setUuid(params.getUuid());
metadata.getSourceInfo().setSourceId(params.getUuid());
if(force || updateSchema) {
if (force) {
//change ownership of metadata to new harvester
metadata.getHarvestInfo().setUuid(params.getUuid());
metadata.getSourceInfo().setSourceId(params.getUuid());
}

if (updateSchema) {
metadata.getDataInfo().setSchemaId(schema);
}

context.getBean(IMetadataManager.class).save(metadata);
}
Expand Down
Loading