feat!: write ordered maps by default (#35)

* feat!: write ordered maps by default

* build!: java client 6.1.6 only supports server 4.9+

* feat: add -um to force unordered maps

Author: dwelch-spike

README.md

@@ -72,6 +72,7 @@ __Options__:
 -tr,--tls-revoke                Revoke certificates identified by their serial number. Values:  serial numbers separated by comma (default: null (Do not revoke certificates))
 -te,--tls-encrypt-only          Enable TLS encryption and disable TLS certificate validation
 -uk,--send-user-key             Send user defined key in addition to hash digest to store on the server. (default: userKey is not sent to reduce meta-data overhead)
+-um,--unorderedMaps             If this flag is present write all maps as unordered maps.
 -u,--usage                      Print usage.
 -v,--verbose                    Verbose mode for debug logging (default: INFO)
 -V,--version                    Print version

doc/releasenotes.md

@@ -59,4 +59,11 @@
 
 ## Aerospike Loader [2.4.3] Release Date [4 Jan 2022]
 ### Fix/Changes:
--   Update log4j to version 2.17.1.
+-   Update log4j to version 2.17.1.
+
+## Aerospike Loader [3.0.0] Release Date [27 Jan 2023]
+### Breaking Changes:
+-   Upgrade Aerospike Java client to 6.1.6 which is only compatible with Aerospike server 4.9 or newer.
+-   Write key ordered maps by default.
+### New features:
+-   Add -um/--unorderedMaps flags to force writing unordered maps.

example/alldatatype.dsv

@@ -1,9 +1,9 @@
 set##key##intData##floatData##stringData##listData##mapData##dateData##blobData##geoData
-set1##key1##11##1.9##string1##["a","b", ["c", "d"]]##{"a": {"b": "c"}}##01/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [123.4, -456.7]}
-set2##key2##12##4.5##string2##["d","e", ["c", "d"]]##{"c": {"b": "c"}}##02/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [123.5, -456.7]}
-set3##key3##13##3.6##string3##["a","b", ["c", "d"]]##{"a": {"b": "c"}}##03/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [123.6, -456.7]}
-set4##key4##14##5.6##string4##["d","e", ["c", "d"]]##{"c": {"b": "c"}}##04/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [123.7, -436.7]}
-set5##key5##15##8.3##string5##["a","b", ["c", "d"]]##{"a": {"b": "c"}}##05/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [123.8, -556.7]}
-set6##key6##16##3.0##string6##["d","e", ["c", "d"]]##{"c": {"b": "c"}}##06/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [123.4, -476.7]}
-set7##key7##17##6.9##string7##["a","b", ["c", "d"]]##{"a": {"b": "c"}}##07/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [133.4, -256.7]}
-set8##key8##18##0.2##string8##["d","b", ["c", "d"]]##{"c": {"b": "c"}}##08/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [143.4, -756.7]}
+set1##key1##11##1.9##string1##["a","b", ["c", "d"]]##{"a": {"b": "c"}}##01/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [123.4, -45.7]}
+set2##key2##12##4.5##string2##["d","e", ["c", "d"]]##{"c": {"b": "c"}}##02/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [123.5, -45.7]}
+set3##key3##13##3.6##string3##["a","b", ["c", "d"]]##{"a": {"b": "c"}}##03/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [123.6, -45.7]}
+set4##key4##14##5.6##string4##["d","e", ["c", "d"]]##{"c": {"b": "c"}}##04/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [123.7, -43.7]}
+set5##key5##15##8.3##string5##["a","b", ["c", "d"]]##{"a": {"b": "c"}}##05/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [123.8, -55.7]}
+set6##key6##16##3.0##string6##["d","e", ["c", "d"]]##{"c": {"b": "c"}}##06/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [123.4, -47.7]}
+set7##key7##17##6.9##string7##["a","b", ["c", "d"]]##{"a": {"b": "c"}}##07/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [133.4, -25.7]}
+set8##key8##18##0.2##string8##["d","b", ["c", "d"]]##{"c": {"b": "c"}}##08/16/2011##6c6f63626c6f623830##{"type": "Point", "coordinates": [143.4, -75.7]}

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>com.aerospike</groupId>
 	<artifactId>aerospike-load</artifactId>
-	<version>2.4.3</version>
+	<version>3.0.0</version>
 	<name>Aerospike DSV Loader</name>
 	<packaging>jar</packaging>
 
@@ -32,7 +32,7 @@
 		<dependency>
 			<groupId>com.aerospike</groupId>
 			<artifactId>aerospike-client</artifactId>
-			<version>4.4.11</version>
+			<version>6.1.6</version>
 		</dependency>
 		<!-- Apache command line parser. -->
 		<dependency>

src/main/java/com/aerospike/load/AerospikeLoad.java

@@ -81,6 +81,7 @@
  * -uk,--send-user-key              Send user defined key in addition to hash digest to store on the server. (default: userKey is not sent to reduce meta-data overhead)
  * -u,--usage           			Print usage.
  * -v,--verbose						Verbose mode for debug logging (default: INFO)
+ * -um,--unordered-map				Write maps as unorderd (default: false)
  * The file names can be a series of file names or directories. 
  *
  * @author Aerospike
@@ -187,6 +188,7 @@ public static void main(String[] args) throws IOException {
 					"Send user defined key in addition to hash digest to store on the server. (default: userKey is not sent to reduce meta-data overhead)"
 					);
 			options.addOption("v", "verbose", false, "Logging all");
+			options.addOption("um", "unorderdMaps", false, "Write all maps as unorderd maps");
 			options.addOption("u", "usage", false, "Print usage.");
 
 			CommandLineParser parser = new PosixParser();

src/main/java/com/aerospike/load/AsWriterTask.java

@@ -28,6 +28,7 @@
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
+import java.util.TreeMap;
 import java.util.concurrent.Callable;
 
 import org.apache.logging.log4j.LogManager;
@@ -461,7 +462,14 @@ private Bin createBinForJson(String binName, String binRawValue) {
 				return new Bin(binName, jsonArray);
 			} else {
 				JSONObject jsonObj = (JSONObject) obj;
-				return  new Bin(binName, jsonObj);
+
+				if (this.params.unorderdMaps) {
+					return  new Bin(binName, jsonObj);
+				}
+
+				TreeMap<?,?> sortedMap = new TreeMap<>();
+				sortedMap.putAll(jsonObj);
+				return  new Bin(binName, sortedMap);
 			}
 
 		} catch (ParseException e) {

src/main/java/com/aerospike/load/Parameters.java

@@ -35,6 +35,7 @@ public class Parameters {
 	long timeZoneOffset;
 	int abortErrorCount;
 	boolean verbose;
+	boolean unorderdMaps;
 
 	/**
 	 * Set parameters from commandline argument. 
@@ -45,6 +46,7 @@ public class Parameters {
 	 * @param timeZoneOffset
 	 * @param abortAtError
 	 * @param verbose
+	 * @param unorderdMaps
 	 */
 	protected Parameters(
 			Host[] hosts,
@@ -53,7 +55,8 @@ protected Parameters(
 			long maxThroughput,
 			long timeZoneOffset,
 			int abortErrorCount,
-			boolean verbose
+			boolean verbose,
+			boolean unorderdMaps
 			) {
 		this.hosts = hosts;
 		this.namespace = namespace;
@@ -62,6 +65,7 @@ protected Parameters(
 		this.timeZoneOffset = timeZoneOffset;
 		this.abortErrorCount = abortErrorCount;
 		this.verbose = verbose;
+		this.unorderdMaps = unorderdMaps;
 	}
 
 	@Override
@@ -71,6 +75,8 @@ public String toString() {
 				", maxThroughput=" + this.maxThroughput +
 				", write-action=" + this.writePolicy.recordExistsAction.toString() +
 				", timeZoneOffset=" + this.timeZoneOffset +
-				", abortErrorCount=" + this.abortErrorCount + "]";
+				", abortErrorCount=" + this.abortErrorCount + 
+				", verbose=" + this.verbose + 
+				", unorderdMaps=" + this.unorderdMaps + "]";
 	}
 }

src/main/java/com/aerospike/load/Utils.java

@@ -138,7 +138,12 @@ protected static Parameters parseParameters(CommandLine cl) throws Exception {
 			verbose = true;
 		}
 
-		return new Parameters(hosts, namespace, writePolicy, maxThroughput, timeZoneOffset, abortErrorCount, verbose);
+		boolean unorderdMaps = false;
+		if (cl.hasOption("um")) {
+			unorderdMaps = true;
+		}
+
+		return new Parameters(hosts, namespace, writePolicy, maxThroughput, timeZoneOffset, abortErrorCount, verbose, unorderdMaps);
 	}
 
 	/**

src/test/java/com/aerospike/load/DataTypeTest.java

@@ -16,6 +16,7 @@
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.SortedMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -35,8 +36,7 @@
 import com.aerospike.client.Key;
 import com.aerospike.client.Record;
 import com.aerospike.client.policy.Policy;
-
-import com.aerospike.load.Parser;
+import com.aerospike.client.cdt.MapOrder;
 
 enum BinType {
 	INTEGER, STRING, BLOB, LIST, MAP, JSON, TIMESTAMP;
@@ -53,6 +53,7 @@ public class DataTypeTest {
 	String port = "3000";
 	String ns = "test";
 	String set = null;
+	MapOrder expectedMapOrder = MapOrder.KEY_ORDERED;
 	//String config = "src/test/resources/allDatatypeCsv.json";
 	String error_count = "0";
 	String write_action = "update";
@@ -434,6 +435,40 @@ public void testStaticBinName() throws Exception {
 		System.out.println("Test static BinName: Complete");
 	}
 
+	//Validate map sort order
+	@Test
+	public void testValidateMapOrder() throws Exception {
+		System.out.println("TestValidateMapOrder: start");
+		if(!client.isConnected()) {
+			System.out.println("Client is not able to connect:" + host + ":" + port);
+			return;
+		}
+		
+		// Create datafile
+
+		HashMap<String, String> binMap = (HashMap<String, String>) testSchema.get("test_map");
+
+		
+		int setMod = 5, range = 100, seed = 10, nrecords = 10;
+		dataFile = rootDir + "dataMap.dsv";
+		writeDataMap(dataFile, nrecords, setMod, range, seed, binMap);
+		
+		// Run Aerospike loader
+		this.expectedMapOrder = MapOrder.UNORDERED;
+		AerospikeLoad.main(new String[]{"-h", host,"-p", port,"-n", ns, "-ec", error_count,"-wa", write_action, "-um", "-c", "src/test/resources/configMap.json", dataFile});
+		
+		// Validate loaded data
+		String dstType = "map";
+		boolean dataValid = validateMap(client, dataFile, nrecords, setMod, range, seed, binMap, dstType);
+		boolean error = getError(log);
+
+		assertTrue(dataValid);
+		assertTrue(!error);
+		this.expectedMapOrder = MapOrder.KEY_ORDERED;
+
+		System.out.println("TestValidateMap: Complete");
+	}
+
 	// Helper functions
 	public void writeDataMap(String fileName, int nrecords, int setMod, int range, int seed,
 			HashMap<String, String> binMap) {
@@ -595,14 +630,28 @@ private boolean validateBin(Key key, Bin bin, String binType, String dstType, Re
 			expected = bin.value.toString().replace("'", "");
 			expected = expected.replace("\"", "");
 
-		} else if (dstType != null && (dstType.equalsIgnoreCase("map")
-				|| dstType.equalsIgnoreCase("json"))) {
-			System.out.println(String.format("Currently json and map can not be matched."));
+		} else if (dstType != null && (dstType.equalsIgnoreCase("json"))) {
+			System.out.println(String.format("Currently json can not be matched."));
 
 			//received = received.toString().replace("=", ":");
 			//expected = bin.value.toString().replace("'", "");
 			//expected = expected.replace("\"", "");
+
 			return true;
+		} else if (dstType != null && (dstType.equalsIgnoreCase("map"))) {
+			System.out.println(String.format("Currently only map order is checked."));
+
+			//received = received.toString().replace("=", ":");
+			//expected = bin.value.toString().replace("'", "");
+			//expected = expected.replace("\"", "");
+
+			MapOrder mapOrder = (received instanceof SortedMap<?,?>)? MapOrder.KEY_ORDERED : MapOrder.UNORDERED;
+
+			if (mapOrder == this.expectedMapOrder) {
+				return true;
+			}
+
+			return false;
 		} else {
 			expected = bin.value.toString();
 		}
@@ -678,7 +727,8 @@ public String getValue(String binName, String binType, int i) {
 			value = "[\"a\", \"b\", \"c\", [\"d\", \"e\"]]";
 			break;
 		case MAP:
-			value = "{\"a\":\"b\", \"b\":\"c\", \"c\":{\"d\":\"e\"}}";
+			value = "{\"a\":\"b\", \"c\":{\"e\":\"d\"}, \"b\":\"c\"}";
+			// sorted value should be "{\"a\":\"b\", \"b\":\"c\", \"c\":{\"d\":\"e\"}}";
 			break;
 		case STRING:
 			if (binName.equalsIgnoreCase("utf8")) {

src/test/resources/dataMap.dsv

@@ -1,11 +1,11 @@
 mapData##set##key
-{"a":"b", "b":"c", "c":{"d":"e"}}##set1##key1
-{"a":"b", "b":"c", "c":{"d":"e"}}##set2##key2
-{"a":"b", "b":"c", "c":{"d":"e"}}##set3##key3
-{"a":"b", "b":"c", "c":{"d":"e"}}##set4##key4
-{"a":"b", "b":"c", "c":{"d":"e"}}##set0##key5
-{"a":"b", "b":"c", "c":{"d":"e"}}##set1##key6
-{"a":"b", "b":"c", "c":{"d":"e"}}##set2##key7
-{"a":"b", "b":"c", "c":{"d":"e"}}##set3##key8
-{"a":"b", "b":"c", "c":{"d":"e"}}##set4##key9
-{"a":"b", "b":"c", "c":{"d":"e"}}##set0##key10
+{"a":"b", "c":{"e":"d"}, "b":"c"}##set1##key1
+{"a":"b", "c":{"e":"d"}, "b":"c"}##set2##key2
+{"a":"b", "c":{"e":"d"}, "b":"c"}##set3##key3
+{"a":"b", "c":{"e":"d"}, "b":"c"}##set4##key4
+{"a":"b", "c":{"e":"d"}, "b":"c"}##set0##key5
+{"a":"b", "c":{"e":"d"}, "b":"c"}##set1##key6
+{"a":"b", "c":{"e":"d"}, "b":"c"}##set2##key7
+{"a":"b", "c":{"e":"d"}, "b":"c"}##set3##key8
+{"a":"b", "c":{"e":"d"}, "b":"c"}##set4##key9
+{"a":"b", "c":{"e":"d"}, "b":"c"}##set0##key10