Add support to get metatada resource json object into the index.

common/src/main/java/org/fao/geonet/utils/Xml.java

@@ -48,16 +48,20 @@
 import org.jdom.transform.JDOMResult;
 import org.jdom.transform.JDOMSource;
 import org.jdom.xpath.XPath;
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 import org.json.XML;
 import org.mozilla.universalchardet.UniversalDetector;
+import org.springframework.util.StringUtils;
+import org.w3c.dom.Node;
 import org.xml.sax.SAXException;
 
 import javax.xml.XMLConstants;
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.Unmarshaller;
 import javax.xml.transform.*;
+import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.sax.SAXResult;
 import javax.xml.transform.stream.StreamResult;
 import javax.xml.transform.stream.StreamSource;
@@ -620,14 +624,26 @@ public static String stripNonValidXMLCharacters(String in) {
     }
 
     public static Element getXmlFromJSON(String jsonAsString) {
+        if (!StringUtils.hasLength(jsonAsString)) {
+            return null;
+        }
         ObjectMapper objectMapper = new ObjectMapper();
         try {
-            JsonNode json = objectMapper.readTree(jsonAsString);
-            String recordAsXml = XML.toString(
-                new JSONObject(
-                    objectMapper.writeValueAsString(json)), "root");
+            JsonNode jsonNode = objectMapper.readTree(jsonAsString);
+
+            String recordAsXml;
+            String jsonString = objectMapper.writeValueAsString(jsonNode);
+            if (jsonAsString.trim().startsWith("[")) {
+                recordAsXml = "<root>" + XML.toString(new JSONArray(jsonString)) + "</root>";
+            } else {
+                recordAsXml = XML.toString(new JSONObject(jsonString), "root");
+            }
             recordAsXml = Xml.stripNonValidXMLCharacters(recordAsXml);
-            return Xml.loadString(recordAsXml, false);
+            if (StringUtils.hasLength(recordAsXml)) {
+                Element xml = Xml.loadString(recordAsXml, false);
+                xml.detach();
+                return xml;
+            }
         } catch (JSONException e) {
             e.printStackTrace();
         } catch (JsonProcessingException e) {
@@ -682,6 +698,24 @@ public static String getString(Document data) {
         return outputter.outputString(data);
     }
 
+    /**
+     *
+     * @param data
+     * @return
+     */
+    public static String getString(Node data) {
+        try {
+            TransformerFactory tf = TransformerFactory.newInstance();
+            Transformer transformer = tf.newTransformer();
+            transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+            StringWriter writer = new StringWriter();
+            transformer.transform(new DOMSource(data), new StreamResult(writer));
+            return writer.toString();
+        } catch (TransformerException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     //---------------------------------------------------------------------------
 
     /**

core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java

@@ -1,5 +1,5 @@
 //=============================================================================
-//===	Copyright (C) 2001-2011 Food and Agriculture Organization of the
+//===	Copyright (C) 2001-2025 Food and Agriculture Organization of the
 //===	United Nations (FAO-UN), United Nations World Food Programme (WFP)
 //===	and United Nations Environment Programme (UNEP)
 //===
@@ -23,6 +23,14 @@
 
 package org.fao.geonet.kernel.datamanager.base;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.PropertyName;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.introspect.Annotated;
+import com.fasterxml.jackson.databind.introspect.AnnotatedMember;
+import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Multimap;
 import com.yammer.metrics.core.TimerContext;
@@ -34,6 +42,7 @@
 import jeeves.xlink.Processor;
 import org.apache.commons.lang.StringUtils;
 import org.fao.geonet.ApplicationContextHolder;
+import org.fao.geonet.annotations.IndexIgnore;
 import org.fao.geonet.api.records.attachments.Store;
 import org.fao.geonet.constants.Geonet;
 import org.fao.geonet.domain.*;
@@ -532,6 +541,9 @@ public void indexMetadata(final String metadataId,
                 int savedCount = userSavedSelectionRepository.countTimesUserSavedMetadata(uuid, 0);
                 fields.put(Geonet.IndexFieldNames.USER_SAVED_COUNT, savedCount);
 
+                // Add metadata file store information
+                fields.putAll(indexMetadataFileStore(fullMd));
+
                 fields.putAll(addExtraFields(fullMd));
 
                 if (fullMd != null) {
@@ -660,4 +672,50 @@ private ServiceContext getServiceContext() {
     public void setApplicationEventPublisher(ApplicationEventPublisher publisher) {
         this.publisher = publisher;
     }
+
+    /**
+     * Get file store properties to be used in the index.
+     *
+     * @return multimap object representing the file store resources to be added to the index.
+     */
+    public Multimap<String, Object> indexMetadataFileStore(AbstractMetadata fullMd) {
+        Multimap<String, Object> indexMetadataFileStoreFields = ArrayListMultimap.create();
+        try {
+            List<MetadataResource> metadataResources = store.getResources(
+                ServiceContext.get(),
+                fullMd.getUuid(),
+                (org.fao.geonet.api.records.attachments.Sort) null,
+                null,
+                !(fullMd instanceof MetadataDraft));
+
+            if (metadataResources != null && !metadataResources.isEmpty()) {
+
+                ObjectMapper objectMapper = new ObjectMapper();
+                objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+                objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+
+                // Exclude fields with the index ignore annotation
+                objectMapper.setAnnotationIntrospector(new JacksonAnnotationIntrospector() {
+                    @Override
+                    public boolean hasIgnoreMarker(AnnotatedMember member) {
+                        return member.hasAnnotation(IndexIgnore.class) || super.hasIgnoreMarker(member);
+                    }
+
+                    @Override
+                    public PropertyName findNameForSerialization(Annotated annotated) {
+                        if (annotated.hasAnnotation(IndexIgnore.class)) return null;
+                        return super.findNameForSerialization(annotated);
+                    }
+                });
+
+                JsonNode jsonNode = objectMapper.valueToTree(metadataResources);
+                indexMetadataFileStoreFields.put("fileStore", jsonNode);
+            }
+        } catch (Exception e) {
+            Log.warning(Geonet.INDEX_ENGINE, String.format(
+                "Resource for metadata '%s'(%d) is not accessible or cannot be found. Skipping index. Error is: %s",
+                fullMd.getUuid(), fullMd.getId(), e.getMessage()));
+        }
+        return indexMetadataFileStoreFields;
+    }
 }

core/src/main/java/org/fao/geonet/kernel/mef/IMEFVisitor.java

@@ -48,4 +48,6 @@ public void handlePublicFile(String file, String changeDate,
 
     public void handlePrivateFile(String file, String changeDate,
                                   InputStream is, int index) throws Exception;
+
+    public void indexMetadata(int index) throws Exception;
 }

core/src/main/java/org/fao/geonet/kernel/mef/Importer.java

@@ -467,6 +467,10 @@ public void handlePrivateFile(String file, String changeDate, InputStream is, in
                 saveFile(context, metadataIdMap.get(index), MetadataResourceVisibility.PRIVATE, file, changeDate, is);
             }
 
+            public void indexMetadata(int index) throws Exception {
+                metadataIndexer.indexMetadata(metadataIdMap.get(index), true, IndexingMode.full);
+            }
+
         });
 
         return metadataIdMap;

core/src/main/java/org/fao/geonet/kernel/mef/MEF2Visitor.java

@@ -107,6 +107,9 @@ public Element handleXml(Path mefFile, IMEFVisitor v) throws Exception {
                             // Handle binaries
                             handleBin(file, v, info, nbMetadata);
 
+                            // Index the record so that the resources are included
+                            v.indexMetadata(nbMetadata);
+
                             nbMetadata++;
                         }
                     }

core/src/main/java/org/fao/geonet/kernel/mef/MEFVisitor.java

@@ -49,6 +49,9 @@ public class MEFVisitor implements IVisitor {
     public void visit(Path mefFile, IMEFVisitor v) throws Exception {
         Element info = handleXml(mefFile, v);
         handleBin(mefFile, v, info, 0);
+
+        // Index the record so that the resources are included
+        v.indexMetadata(0);
     }
 
     // --------------------------------------------------------------------------

core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java

@@ -221,15 +221,24 @@ private void addMDFields(Element doc, Path schemaDir,
     }
 
     private void addMoreFields(Element doc, Multimap<String, Object> fields) {
-        ArrayList<String> objectFields = Lists.newArrayList(INDEXING_ERROR_MSG);
-        fields.entries().forEach(e -> {
-            Element newElement = new Element(e.getKey())
-                .setText(String.valueOf(e.getValue()));
-            if (objectFields.contains(e.getKey())) {
-                newElement.setAttribute("type", "object");
-            }
-            doc.addContent(newElement);
-        });
+        fields.entries().forEach(e -> doc.addContent(getNewElement(e.getKey(), e.getValue())
+            .setText(String.valueOf(e.getValue()))));
+    }
+
+    /**
+     * Create a new element with the key as name. If the object is a JsonNode, we set the type attribute to object
+     * Added the type as object will cause it to keep the json structure
+     * @param key the name of the element
+     * @param obj object to check if it is a JsonNode
+     * @return new Element with the key as name
+     */
+    // If the object is a JsonNode, we set the type attribute to object
+    private Element getNewElement(String key, Object obj) {
+        Element element =  new Element(key);
+        if (obj instanceof JsonNode) {
+            element.setAttribute("type", "object");
+        }
+        return element;
     }
 
     public Element makeField(String name, String value) {

domain/src/main/java/org/fao/geonet/annotations/IndexIgnore.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2001-2025 Food and Agriculture Organization of the
+ * United Nations (FAO-UN), United Nations World Food Programme (WFP)
+ * and United Nations Environment Programme (UNEP)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or (at
+ * your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+ *
+ * Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
+ * Rome - Italy. email: [email protected]
+ */
+
+package org.fao.geonet.annotations;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation to indicate that a field/method should be excluded in the index.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.FIELD, ElementType.METHOD})
+public @interface IndexIgnore {
+}

domain/src/main/java/org/fao/geonet/domain/MetadataResource.java

@@ -1,6 +1,6 @@
 /*
  * =============================================================================
- * ===	Copyright (C) 2001-2016 Food and Agriculture Organization of the
+ * ===	Copyright (C) 2001-2025 Food and Agriculture Organization of the
  * ===	United Nations (FAO-UN), United Nations World Food Programme (WFP)
  * ===	and United Nations Environment Programme (UNEP)
  * ===
@@ -25,6 +25,9 @@
 
 package org.fao.geonet.domain;
 
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import org.fao.geonet.annotations.IndexIgnore;
+
 import java.util.Date;
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
@@ -35,10 +38,13 @@
  */
 @XmlRootElement(name = "resource")
 @XmlAccessorType(XmlAccessType.FIELD)
+@JsonPropertyOrder(alphabetic=true)
 public interface MetadataResource {
 
+    @IndexIgnore
     String getId();
 
+    // Don't ignore url in the index as it is used to link the urls
     String getUrl();
 
     MetadataResourceVisibility getVisibility();
@@ -47,12 +53,16 @@ public interface MetadataResource {
 
     Date getLastModification();
 
+    @IndexIgnore
     String getFilename();
 
+    @IndexIgnore
     boolean isApproved();
 
+    @IndexIgnore // Metadata id is already in the index so no need to add it again.
     int getMetadataId();
 
+    @IndexIgnore // Metadata uuid is already in the index so no need to add it again.
     String getMetadataUuid();
 
     String getVersion();

harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/BaseGeoNetworkAligner.java

@@ -370,6 +370,9 @@ public void handlePrivateFile(String file,
                             handleFile(id, file, MetadataResourceVisibility.PRIVATE, changeDate, is, privateFiles[index]);
                         }
 
+                        public void indexMetadata(int index) throws Exception {
+                            metadataIndexer.indexMetadata(id, true, IndexingMode.full);
+                        }
                     });
                 } catch (Exception e) {
                     //--- we ignore the exception here. Maybe the metadata has been removed just now
@@ -875,6 +878,10 @@ public void handlePrivateFile(String file, String changeDate,
                         handleFile(file, changeDate, is, index, MetadataResourceVisibility.PRIVATE);
                     }
                 }
+
+                public void indexMetadata(int index) throws Exception {
+                    metadataIndexer.indexMetadata(id[index], true, IndexingMode.full);
+                }
             });
         } catch (Exception e) {
             //--- we ignore the exception here. Maybe the metadata has been removed just now

schemas/iso19115-3.2018/src/test/java/org/fao/geonet/schema/XslConversionTest.java

@@ -30,6 +30,8 @@
 import org.fao.geonet.utils.Xml;
 import org.jdom.Element;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+
 import org.junit.Test;
 import org.xmlunit.builder.DiffBuilder;
 import org.xmlunit.builder.Input;
@@ -72,4 +74,22 @@ public void testOdsConversion() throws Exception {
             String.format("Differences: %s", diff.toString()),
             diff.hasDifferences());
     }
+
+    @Test
+    public void testArrayConversion() throws Exception {
+        String jsonString = "[{\"Test\":\"value1\"},{\"Test\":\"value2\"}]";
+        Element xmlFromJSON = Xml.getXmlFromJSON(jsonString);
+
+        String expectedElement = "<root><array><Test>value1</Test></array><array><Test>value2</Test></array></root>";
+
+        Diff diff = DiffBuilder
+            .compare(Input.fromString(expectedElement))
+            .withTest(Input.fromString(Xml.getString(xmlFromJSON)))
+            .withNodeMatcher(new DefaultNodeMatcher(ElementSelectors.byName))
+            .normalizeWhitespace()
+            .ignoreComments()
+            .checkForSimilar()
+            .build();
+        assertFalse(diff.toString(), diff.hasDifferences());
+    }
 }