Gigamp, IndexerMultiValue, CSV Export - unsafe memory access operation

[zdenek-jonas]

By Combination of IndexerMultiValue index and CSV export is an Exception raised:

Exception in thread "main" java.lang.InternalError: a fault occurred in an unsafe memory access operation
	at org.eclipse.store.storage.types.StorageDataConverterTypeBinaryToCsv$UTF8.write_bytes(StorageDataConverterTypeBinaryToCsv.java:912)
	at org.eclipse.store.storage.types.StorageDataConverterTypeBinaryToCsv$UTF8$13.writeValue(StorageDataConverterTypeBinaryToCsv.java:1068)
	at org.eclipse.store.storage.types.StorageDataConverterTypeBinaryToCsv$UTF8.processEntity(StorageDataConverterTypeBinaryToCsv.java:515)
	at org.eclipse.store.storage.types.StorageDataConverterTypeBinaryToCsv$UTF8.accept(StorageDataConverterTypeBinaryToCsv.java:1100)
	at org.eclipse.store.storage.types.StorageDataFileItemIterator$Default.processBufferedEntities(StorageDataFileItemIterator.java:298)
	at org.eclipse.store.storage.types.StorageDataFileItemIterator$Default.processInputFile(StorageDataFileItemIterator.java:244)
	at org.eclipse.store.storage.types.StorageDataConverterTypeBinaryToCsv$UTF8.convertDataFile(StorageDataConverterTypeBinaryToCsv.java:641)
	at test.microstream.gigamap.indexer.MultipleValueIndexTest.lambda$convertToCSV$0(MultipleValueIndexTest.java:157)
	at java.base/java.lang.Iterable.forEach(Iterable.java:75)
	at test.microstream.gigamap.indexer.MultipleValueIndexTest.convertToCSV(MultipleValueIndexTest.java:157)
	at test.microstream.gigamap.indexer.MultipleValueIndexTest.exportAsCSV(MultipleValueIndexTest.java:168)
	at test.microstream.gigamap.indexer.MultipleValueIndexTest.main(MultipleValueIndexTest.java:58)

Following code:

package test.microstream.gigamap.indexer;

/*-
 * #%L
 * EclipseStore GigaMap
 * %%
 * Copyright (C) 2023 - 2025 MicroStream Software
 * %%
 * This program and the accompanying materials are made
 * available under the terms of the Eclipse Public License 2.0
 * which is available at https://www.eclipse.org/legal/epl-2.0/
 *
 * SPDX-License-Identifier: EPL-2.0
 * #L%
 */

import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;

import org.eclipse.serializer.collections.types.XSequence;
import org.eclipse.serializer.util.cql.CQL;
import org.eclipse.store.afs.nio.types.NioFileSystem;
import org.eclipse.store.gigamap.types.GigaMap;
import org.eclipse.store.gigamap.types.IndexerMultiValue;
import org.eclipse.store.storage.embedded.types.EmbeddedStorage;
import org.eclipse.store.storage.embedded.types.EmbeddedStorageManager;
import org.eclipse.store.storage.types.*;


public class MultipleValueIndexTest
{

	public static void main(String[] args) throws Exception
	{
		Path dataDirectory = Path.of("target", "multiValue-storage");
		Path exportDirectory = Path.of("target", "multiValue-export");

		GigaMap<Patient> gigaMap = GigaMap.New();
		PatientIdentifierIndexer patientIdentifierIndexer = new PatientIdentifierIndexer();
		gigaMap.index().bitmap().add(patientIdentifierIndexer);
		Patient patient1 = new Patient("John", 25, List.of("123", "456"));
		Patient patient2 = new Patient("Jane", 30, List.of("123", "789"));
		gigaMap.addAll(patient1, patient2);

		gigaMap.update(patient1, p -> {
			p.setIdentifiers(List.of("123", "abc"));
		});


		try (EmbeddedStorageManager storage = EmbeddedStorage.start(gigaMap, dataDirectory)) {
			exportAsCSV(storage, exportDirectory); //<== 
		}

	}


	private static class PatientIdentifierIndexer extends IndexerMultiValue.Abstract<Patient, String>
	{

		@Override
		public Iterable<? extends String> indexEntityMultiValue(Patient entity)
		{
			return entity.getIdentifiers();
		}

		@Override
		public Class<String> keyType()
		{
			return String.class;
		}
	}

	private static class Patient
	{
		private String name;
		private final int age;
		private List<String> identifiers;

		public Patient(String name, int age, List<String> identifiers)
		{
			this.name = name;
			this.age = age;
			this.identifiers = identifiers;
		}

		public String getName()
		{
			return name;
		}

		public int getAge()
		{
			return age;
		}

		public List<String> getIdentifiers()
		{
			return identifiers;
		}

		public void setIdentifiers(final List<String> identifiers)
		{
			this.identifiers = identifiers;
		}

		public void setName(String name)
		{
			this.name = name;
		}
	}

	public static XSequence<Path> export(final EmbeddedStorageManager storage, final String target)
	{

		final NioFileSystem fileSystem = NioFileSystem.New();

		final String fileSuffix = "bin";
		final StorageConnection connection = storage.createConnection();
		final StorageEntityTypeExportStatistics exportResult = connection.exportTypes(
				new StorageEntityTypeExportFileProvider.Default(
						fileSystem.ensureDirectoryPath(target),
						fileSuffix
				),
				typeHandler -> true // export all, customize if necessary
		);
		final XSequence<Path> exportFiles = CQL
				.from(exportResult.typeStatistics().values())
				.project(s -> Paths.get(s.file().toPathString()))
				.execute();

		return exportFiles;
	}

	public static void convertToCSV(final EmbeddedStorageManager storage, final XSequence<Path> files, final String target)
	{

		final NioFileSystem fileSystem = NioFileSystem.New();

		final StorageDataConverterTypeBinaryToCsv converter =
				new StorageDataConverterTypeBinaryToCsv.UTF8(
						StorageDataConverterCsvConfiguration.defaultConfiguration(),
						new StorageEntityTypeConversionFileProvider.Default(
								fileSystem.ensureDirectoryPath(target),
								"csv"
						),
						storage.typeDictionary(),
						null, // no type name mapping
						4096, // read buffer size
						4096  // write buffer size
				);

		files.forEach(file -> converter.convertDataFile(
				fileSystem.ensureFile(file).tryUseReading()
		));
	}

	public static void exportAsCSV(final EmbeddedStorageManager storage, final Path targetDir)
	{

		final Path binDir = targetDir.resolve("bin");
		final XSequence<Path> exports = export(storage, binDir.toString());

		final Path csvDir = targetDir.resolve("csv");
		convertToCSV(storage, exports, csvDir.toString());

	}
}