From 54f54bb0b2cb9480407939d123abc95129c1f6b7 Mon Sep 17 00:00:00 2001 From: Rrutum <71127900+rrutum@users.noreply.github.com> Date: Mon, 14 Jul 2025 18:52:04 +0530 Subject: [PATCH 1/9] Migrate simple-json to jackson --- pom.xml | 13 +- .../java/com/aerospike/load/AsWriterTask.java | 49 +++-- src/main/java/com/aerospike/load/Parser.java | 188 +++++++++--------- .../com/aerospike/load/RelaxedJsonMapper.java | 181 +++++++++++++++++ .../java/com/aerospike/load/DataTypeTest.java | 58 +++--- .../aerospike/load/RelaxedJsonMapperTest.java | 87 ++++++++ 6 files changed, 425 insertions(+), 151 deletions(-) create mode 100644 src/main/java/com/aerospike/load/RelaxedJsonMapper.java create mode 100644 src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java diff --git a/pom.xml b/pom.xml index 2f69d97..08f27b8 100644 --- a/pom.xml +++ b/pom.xml @@ -58,11 +58,16 @@ 4.13.1 test - + - com.googlecode.json-simple - json-simple - 1.1.1 + com.fasterxml.jackson.core + jackson-core + 2.15.2 + + + com.fasterxml.jackson.core + jackson-databind + 2.15.2 diff --git a/src/main/java/com/aerospike/load/AsWriterTask.java b/src/main/java/com/aerospike/load/AsWriterTask.java index f4353c4..ad75d87 100644 --- a/src/main/java/com/aerospike/load/AsWriterTask.java +++ b/src/main/java/com/aerospike/load/AsWriterTask.java @@ -33,10 +33,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import java.util.Map; import com.aerospike.client.AerospikeClient; import com.aerospike.client.AerospikeException; @@ -69,7 +68,7 @@ public class AsWriterTask implements Callable { private Parameters params; private Counter counters; - private JSONParser jsonParser; + // JSON parsing is now handled by RelaxedJsonMapper private static Logger log = LogManager.getLogger(AsWriterTask.class); @@ -229,14 +228,9 @@ private Key getKeyAndBinsFromDataline(List bins) { + " Aerospike Bin processing Error: " + ae.getResultCode()); handleProcessLineError(ae); - } catch (ParseException pe) { - - log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + " Parsing Error: " + pe); - handleProcessLineError(pe); - } catch (Exception e) { - log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + " Unknown Error: " + e); + log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + " Error: " + e); handleProcessLineError(e); } @@ -247,7 +241,7 @@ private Key getKeyAndBinsFromDataline(List bins) { * Validate if number of column in data line are same as provided in config file. * Throw exception the more columns are present then given. */ - private void validateNColumnInDataline() throws ParseException { + private void validateNColumnInDataline() throws Exception { // Throw exception if n_columns(datafile) are more than n_columns(configfile). int n_column = Integer.parseInt(dsvConfigs.get(Constants.N_COLUMN)); @@ -258,7 +252,7 @@ private void validateNColumnInDataline() throws ParseException { log.warn("File: " + Utils.getFileName(fileName) + " Line: " + lineNumber + " Number of column mismatch:Columns in data file is less than number of column in config file."); } else { - throw new ParseException(lineNumber); + throw new Exception("Column count mismatch at line " + lineNumber); } } @@ -451,28 +445,31 @@ private Bin createBinForJson(String binName, String binRawValue) { try { log.debug(binRawValue); - if (jsonParser == null) { - jsonParser = new JSONParser(); - } - - Object obj = jsonParser.parse(binRawValue); + JsonNode jsonNode = RelaxedJsonMapper.parseJson(binRawValue); - if (obj instanceof JSONArray) { - JSONArray jsonArray = (JSONArray) obj; + if (jsonNode.isArray()) { + List jsonArray = RelaxedJsonMapper.parseJsonToList(binRawValue); return new Bin(binName, jsonArray); } else { - JSONObject jsonObj = (JSONObject) obj; + Object jsonObj = RelaxedJsonMapper.jsonNodeToObject(jsonNode); if (this.params.unorderdMaps) { - return new Bin(binName, jsonObj); + if (jsonObj instanceof Map) { + return new Bin(binName, (Map) jsonObj); + } + return new Bin(binName, jsonObj.toString()); } - TreeMap sortedMap = new TreeMap<>(); - sortedMap.putAll(jsonObj); - return new Bin(binName, sortedMap); + if (jsonObj instanceof Map) { + TreeMap sortedMap = new TreeMap<>(); + sortedMap.putAll((Map) jsonObj); + return new Bin(binName, sortedMap); + } + + return new Bin(binName, jsonObj.toString()); } - } catch (ParseException e) { + } catch (IOException e) { log.error("Failed to parse JSON: " + e); return null; } diff --git a/src/main/java/com/aerospike/load/Parser.java b/src/main/java/com/aerospike/load/Parser.java index 7d5a6e5..c8f28a4 100644 --- a/src/main/java/com/aerospike/load/Parser.java +++ b/src/main/java/com/aerospike/load/Parser.java @@ -32,10 +32,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; /** * Parser class to parse different schema/(data definition) and data files. @@ -58,21 +56,16 @@ public class Parser { public static boolean parseJSONColumnDefinitions(File configFile, HashMap dsvConfigs, List mappingDefs) { FileReader fr = null; try { - JSONParser jsonParser = new JSONParser(); - - Object obj; fr = new FileReader(configFile); - obj = jsonParser.parse(fr); + JsonNode jobj = RelaxedJsonMapper.parseJson(fr); - JSONObject jobj; - if (obj == null) { + if (jobj == null) { log.error("Empty config File."); if (fr != null) fr.close(); return false; } else { - jobj = (JSONObject) obj; - log.debug("Config file contents: " + jobj.toJSONString()); + log.debug("Config file contents: " + RelaxedJsonMapper.jsonNodeToString(jobj)); } /* @@ -91,12 +84,6 @@ public static boolean parseJSONColumnDefinitions(File configFile, HashMap dsvConfigs) { + private static boolean getUpdateDsvConfig(JsonNode jobj, HashMap dsvConfigs) { Object obj = null; // Get Metadata of loader (update dsvConfigs) - if ((obj = getFromJsonObject(jobj, Constants.VERSION)) == null) { + if ((obj = RelaxedJsonMapper.getFromJsonNode(jobj, Constants.VERSION)) == null) { log.error("\"" + Constants.VERSION + "\" Key is missing in config file."); return false; } @@ -134,23 +121,23 @@ private static boolean getUpdateDsvConfig(JSONObject jobj, HashMap mappingDefs) throws Exception { + private static boolean getUpdateMappingColumnDefs(JsonNode jobj, List mappingDefs) throws Exception { Object obj = null; - JSONArray mappings; - if ((obj = getFromJsonObject(jobj, Constants.MAPPINGS)) != null) { - mappings = (JSONArray) obj; - Iterator it = mappings.iterator(); - while (it.hasNext()) { - JSONObject mappingObj = (JSONObject) it.next(); - MappingDefinition md = getMappingDef(mappingObj); - if (md != null) { - mappingDefs.add(md); - } else { - log.error("Error in parsing mappingdef: " + mappingObj.toString()); - return false; + JsonNode mappings; + if ((obj = RelaxedJsonMapper.getFromJsonNode(jobj, Constants.MAPPINGS)) != null) { + mappings = jobj.get(Constants.MAPPINGS); + if (mappings.isArray()) { + for (JsonNode mappingObj : mappings) { + MappingDefinition md = getMappingDef(mappingObj); + if (md != null) { + mappingDefs.add(md); + } else { + log.error("Error in parsing mappingdef: " + RelaxedJsonMapper.jsonNodeToString(mappingObj)); + return false; + } } } } return true; } - private static Object getFromJsonObject(JSONObject jobj, String key) { - return jobj.get(key); + private static Object getFromJsonObject(JsonNode jobj, String key) { + return RelaxedJsonMapper.getFromJsonNode(jobj, key); } /* * Parse mapping definition from config file to get mappingDef * This will have (secondary_mapping_ keyDefinition, setDefinition, binDefinition). */ - private static MappingDefinition getMappingDef(JSONObject mappingObj) throws Exception { + private static MappingDefinition getMappingDef(JsonNode mappingObj) throws Exception { boolean secondary_mapping = false; MetaDefinition keyColumnDef = null; @@ -203,33 +190,33 @@ private static MappingDefinition getMappingDef(JSONObject mappingObj) throws Exc } if ((obj = getFromJsonObject(mappingObj, Constants.KEY)) != null) { - keyColumnDef = getMetaDefs((JSONObject) obj, Constants.KEY); + keyColumnDef = getMetaDefs(mappingObj.get(Constants.KEY), Constants.KEY); } else { - log.error("\"" + Constants.KEY + "\" Key is missing in mapping. Mapping: " + mappingObj.toString()); + log.error("\"" + Constants.KEY + "\" Key is missing in mapping. Mapping: " + RelaxedJsonMapper.jsonNodeToString(mappingObj)); return null; } if ((obj = getFromJsonObject(mappingObj, Constants.SET)) == null) { - log.error("\"" + Constants.SET + "\" Key is missing in mapping. Mapping: " + mappingObj.toString()); + log.error("\"" + Constants.SET + "\" Key is missing in mapping. Mapping: " + RelaxedJsonMapper.jsonNodeToString(mappingObj)); return null; } else if (obj instanceof String) { setColumnDef = new MetaDefinition(obj.toString(), null); } else { - setColumnDef = getMetaDefs((JSONObject) obj, Constants.SET); + setColumnDef = getMetaDefs(mappingObj.get(Constants.SET), Constants.SET); } if ((obj = getFromJsonObject(mappingObj, Constants.BINLIST)) != null) { - JSONArray binObjList = (JSONArray) obj; - Iterator it = binObjList.iterator(); - while (it.hasNext()) { - JSONObject binObj = (JSONObject) it.next(); - BinDefinition binDef = getBinDefs(binObj); - if (binDef != null) { - binColumnDefs.add(binDef); - } else { - log.error("Error in parsing binDef: " + binObj.toString()); - return null; + JsonNode binObjList = mappingObj.get(Constants.BINLIST); + if (binObjList.isArray()) { + for (JsonNode binObj : binObjList) { + BinDefinition binDef = getBinDefs(binObj); + if (binDef != null) { + binColumnDefs.add(binDef); + } else { + log.error("Error in parsing binDef: " + RelaxedJsonMapper.jsonNodeToString(binObj)); + return null; + } } } } else { @@ -243,7 +230,7 @@ private static MappingDefinition getMappingDef(JSONObject mappingObj) throws Exc /* * Parsing Meta definition(for Set or Key) from config file and populate metaDef object. */ - private static MetaDefinition getMetaDefs(JSONObject jobj, String jobjName) { + private static MetaDefinition getMetaDefs(JsonNode jobj, String jobjName) { // Parsing Key, Set definition ColumnDefinition valueDef = new ColumnDefinition(-1, null, null, null, null, null, null); @@ -253,14 +240,15 @@ private static MetaDefinition getMetaDefs(JSONObject jobj, String jobjName) { } else if ((jobj.get(Constants.COLUMN_NAME)) != null) { - valueDef.columnName = (String) (jobj.get(Constants.COLUMN_NAME)); + valueDef.columnName = jobj.get(Constants.COLUMN_NAME).asText(); } else { log.error("Column_name or pos info is missing. Specify proper key/set mapping in config file for: " + jobjName + ":" - + jobj.toString()); + + RelaxedJsonMapper.jsonNodeToString(jobj)); } - valueDef.setSrcType((String) jobj.get(Constants.TYPE)); + JsonNode typeNode = jobj.get(Constants.TYPE); + valueDef.setSrcType(typeNode != null ? typeNode.asText() : null); // Default set type is 'string'. what is default key type? if (Constants.SET.equalsIgnoreCase(jobjName) && valueDef.srcType == null) { @@ -268,8 +256,9 @@ private static MetaDefinition getMetaDefs(JSONObject jobj, String jobjName) { } // Get prefix to remove. Prefix will be removed from data - if ((jobj.get(Constants.REMOVE_PREFIX)) != null) { - valueDef.removePrefix = (String) jobj.get(Constants.REMOVE_PREFIX); + JsonNode prefixNode = jobj.get(Constants.REMOVE_PREFIX); + if (prefixNode != null) { + valueDef.removePrefix = prefixNode.asText(); } return new MetaDefinition(null, valueDef); @@ -278,7 +267,7 @@ private static MetaDefinition getMetaDefs(JSONObject jobj, String jobjName) { /* * Parsing Bin definition from config file and populate BinDef object */ - private static BinDefinition getBinDefs(JSONObject jobj) { + private static BinDefinition getBinDefs(JsonNode jobj) { /* * Sample Bin object * {"name": "age", "value": {"column_name": "age", "type" : "integer"} } @@ -290,23 +279,23 @@ private static BinDefinition getBinDefs(JSONObject jobj) { ColumnDefinition nameDef = new ColumnDefinition(-1, null, null, null, null, null, null); String staticBinName = null; - if ((obj = jobj.get(Constants.NAME)) == null) { - log.error(Constants.NAME + " key is missing object: " + jobj.toString()); + JsonNode nameNode = jobj.get(Constants.NAME); + if (nameNode == null) { + log.error(Constants.NAME + " key is missing object: " + RelaxedJsonMapper.jsonNodeToString(jobj)); return null; - } else if (!(obj instanceof JSONObject)) { - staticBinName = (String) obj; + } else if (nameNode.isTextual()) { + staticBinName = nameNode.asText(); } else { - JSONObject nameObj = (JSONObject) obj; - if ((nameObj.get(Constants.COLUMN_POSITION)) != null) { - - nameDef.columnPos = (Integer.parseInt(nameObj.get(Constants.COLUMN_POSITION).toString()) - 1); - - } else if ((nameObj.get(Constants.COLUMN_NAME)) != null) { - - nameDef.columnName = (String) (nameObj.get(Constants.COLUMN_NAME)); - + JsonNode posNode = nameNode.get(Constants.COLUMN_POSITION); + if (posNode != null) { + nameDef.columnPos = (Integer.parseInt(posNode.asText()) - 1); } else { - log.error("Column_name or pos info is missing. Specify proper bin name mapping in config file for: " + jobj.toString()); + JsonNode colNameNode = nameNode.get(Constants.COLUMN_NAME); + if (colNameNode != null) { + nameDef.columnName = colNameNode.asText(); + } else { + log.error("Column_name or pos info is missing. Specify proper bin name mapping in config file for: " + RelaxedJsonMapper.jsonNodeToString(jobj)); + } } } @@ -314,33 +303,40 @@ private static BinDefinition getBinDefs(JSONObject jobj) { ColumnDefinition valueDef = new ColumnDefinition(-1, null, null, null, null, null, null); String staticBinValue = null; - if ((obj = jobj.get(Constants.VALUE)) == null) { - log.error(Constants.VALUE + " key is missing in bin object:" + jobj.toString()); + JsonNode valueNode = jobj.get(Constants.VALUE); + if (valueNode == null) { + log.error(Constants.VALUE + " key is missing in bin object:" + RelaxedJsonMapper.jsonNodeToString(jobj)); return null; - } else if (!(obj instanceof JSONObject)) { - staticBinValue = (String) obj; + } else if (valueNode.isTextual()) { + staticBinValue = valueNode.asText(); } else { - JSONObject valueObj = (JSONObject) obj; - - if ((valueObj.get(Constants.COLUMN_POSITION)) != null) { - - valueDef.columnPos = (Integer.parseInt(valueObj.get(Constants.COLUMN_POSITION).toString()) - 1); - - } else if ((valueObj.get(Constants.COLUMN_NAME)) != null) { - - valueDef.columnName = (String) (valueObj.get(Constants.COLUMN_NAME)); - + JsonNode posNode = valueNode.get(Constants.COLUMN_POSITION); + if (posNode != null) { + valueDef.columnPos = (Integer.parseInt(posNode.asText()) - 1); } else { - log.error("Column_name or pos info is missing. Specify proper bin value mapping in config file for: " + jobj.toString()); + JsonNode colNameNode = valueNode.get(Constants.COLUMN_NAME); + if (colNameNode != null) { + valueDef.columnName = colNameNode.asText(); + + } else { + log.error("Column_name or pos info is missing. Specify proper bin value mapping in config file for: " + RelaxedJsonMapper.jsonNodeToString(jobj)); + } } - valueDef.setSrcType((String) (valueObj.get(Constants.TYPE))); + JsonNode typeNode = valueNode.get(Constants.TYPE); + valueDef.setSrcType(typeNode != null ? typeNode.asText() : null); if (valueDef.srcType == null) { - log.error(Constants.TYPE + " key is missing in bin object: " + jobj.toString()); + log.error(Constants.TYPE + " key is missing in bin object: " + RelaxedJsonMapper.jsonNodeToString(jobj)); } - valueDef.setDstType((String) (valueObj.get(Constants.DST_TYPE))); - valueDef.encoding = (String) (valueObj.get(Constants.ENCODING)); - valueDef.removePrefix = ((String) (valueObj.get(Constants.REMOVE_PREFIX))); + + JsonNode dstTypeNode = valueNode.get(Constants.DST_TYPE); + valueDef.setDstType(dstTypeNode != null ? dstTypeNode.asText() : null); + + JsonNode encodingNode = valueNode.get(Constants.ENCODING); + valueDef.encoding = encodingNode != null ? encodingNode.asText() : null; + + JsonNode prefixNode = valueNode.get(Constants.REMOVE_PREFIX); + valueDef.removePrefix = prefixNode != null ? prefixNode.asText() : null; } return new BinDefinition(staticBinName, staticBinValue, nameDef, valueDef); diff --git a/src/main/java/com/aerospike/load/RelaxedJsonMapper.java b/src/main/java/com/aerospike/load/RelaxedJsonMapper.java new file mode 100644 index 0000000..b6f8db5 --- /dev/null +++ b/src/main/java/com/aerospike/load/RelaxedJsonMapper.java @@ -0,0 +1,181 @@ +package com.aerospike.load; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.FileReader; +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Utility class for parsing JSON with relaxed syntax using Jackson. + * Supports unquoted field names, single quotes, and automatic type coercion. + */ +public class RelaxedJsonMapper { + + private static final ObjectMapper RELAXED_MAPPER = new ObjectMapper(); + private static final ObjectMapper STANDARD_MAPPER = new ObjectMapper(); + + static { + // Configure relaxed mapper to allow JSON supersets + RELAXED_MAPPER.enable(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES); + RELAXED_MAPPER.enable(JsonParser.Feature.ALLOW_SINGLE_QUOTES); + RELAXED_MAPPER.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + // Use regular integers instead of BigInteger for better compatibility + RELAXED_MAPPER.disable(DeserializationFeature.USE_BIG_INTEGER_FOR_INTS); + } + + /** + * Parse JSON from a FileReader. + * @param reader FileReader containing JSON data + * @return JsonNode representing the parsed JSON + * @throws IOException if parsing fails + */ + public static JsonNode parseJson(FileReader reader) throws IOException { + return RELAXED_MAPPER.readTree(reader); + } + + /** + * Parse JSON from a string. + * @param json JSON string + * @return JsonNode representing the parsed JSON + * @throws IOException if parsing fails + */ + public static JsonNode parseJson(String json) throws IOException { + return RELAXED_MAPPER.readTree(json); + } + + /** + * Parse JSON string into a Map. + * @param json JSON string + * @return Map representation of the JSON + * @throws IOException if parsing fails + */ + public static Map parseJsonToMap(String json) throws IOException { + return RELAXED_MAPPER.readValue(json, new TypeReference>() {}); + } + + /** + * Parse JSON string into a List. + * @param json JSON string + * @return List representation of the JSON + * @throws IOException if parsing fails + */ + public static List parseJsonToList(String json) throws IOException { + return RELAXED_MAPPER.readValue(json, new TypeReference>() {}); + } + + /** + * Parse JSON with automatic key type coercion (similar to the sample code). + * This method converts string keys to appropriate types (Integer, Long, Double, Boolean). + * @param json JSON string + * @return Map with coerced keys + * @throws IOException if parsing fails + */ + public static Map parseJsonWithKeyCoercion(String json) throws IOException { + // First pass: everything is still strings for keys + Map intermediate = RELAXED_MAPPER.readValue(json, new TypeReference>() {}); + + // Second pass: coerce keys to proper types + Map result = new LinkedHashMap<>(); + for (Map.Entry entry : intermediate.entrySet()) { + result.put(coerceKey(entry.getKey()), entry.getValue()); + } + return result; + } + + /** + * Tries to turn a textual key into Integer, Long, Double, or Boolean. + * Falls back to the original String if none match. + * @param key The string key to coerce + * @return The coerced key or original string if no coercion is possible + */ + private static Object coerceKey(String key) { + try { + return Integer.valueOf(key); + } catch (NumberFormatException ignored) {} + + try { + return Long.valueOf(key); + } catch (NumberFormatException ignored) {} + + try { + return Double.valueOf(key); + } catch (NumberFormatException ignored) {} + + try { + return Float.valueOf(key); + } catch (NumberFormatException ignored) {} + + if ("true".equalsIgnoreCase(key) || "false".equalsIgnoreCase(key)) { + return Boolean.valueOf(key); + } + + return key; // leave it a String + } + + /** + * Convert a JsonNode to a Java object (Map, List, or primitive). + * @param node The JsonNode to convert + * @return The converted Java object + */ + public static Object jsonNodeToObject(JsonNode node) { + if (node == null || node.isNull()) { + return null; + } else if (node.isBoolean()) { + return node.asBoolean(); + } else if (node.isInt()) { + return node.asInt(); + } else if (node.isLong()) { + return node.asLong(); + } else if (node.isFloat()) { + return node.floatValue(); + } else if (node.isDouble()) { + return node.asDouble(); + } else if (node.isTextual()) { + return node.asText(); + } else if (node.isArray()) { + return STANDARD_MAPPER.convertValue(node, List.class); + } else if (node.isObject()) { + return STANDARD_MAPPER.convertValue(node, Map.class); + } + return node.toString(); + } + + /** + * Get a field value from a JsonNode. + * @param node The JsonNode to get the field from + * @param fieldName The name of the field + * @return The field value as Object, or null if not found + */ + public static Object getFromJsonNode(JsonNode node, String fieldName) { + if (node == null || !node.has(fieldName)) { + return null; + } + return jsonNodeToObject(node.get(fieldName)); + } + + /** + * Check if a JsonNode has a specific field. + * @param node The JsonNode to check + * @param fieldName The name of the field + * @return true if the field exists, false otherwise + */ + public static boolean hasField(JsonNode node, String fieldName) { + return node != null && node.has(fieldName); + } + + /** + * Get a JsonNode as a string representation. + * @param node The JsonNode to convert + * @return String representation of the JsonNode + */ + public static String jsonNodeToString(JsonNode node) { + return node != null ? node.toString() : null; + } +} \ No newline at end of file diff --git a/src/test/java/com/aerospike/load/DataTypeTest.java b/src/test/java/com/aerospike/load/DataTypeTest.java index dd2eb3c..86a8506 100644 --- a/src/test/java/com/aerospike/load/DataTypeTest.java +++ b/src/test/java/com/aerospike/load/DataTypeTest.java @@ -23,9 +23,8 @@ import java.util.Map.Entry; import java.util.Random; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -64,7 +63,7 @@ public class DataTypeTest { String testSchemaFile = "src/test/resources/testSchema.json"; // String dataFile = "src/test/resources/data.csv"; String log = "aerospike-load.log"; - JSONObject testSchema = null; + JsonNode testSchema = null; AerospikeClient client; @Before @@ -105,18 +104,28 @@ public List> parseDataFile(String dataFile) { return recordDataList; } - public JSONObject parseConfigFile(String configFile) { - JSONParser parser = new JSONParser(); - JSONObject jsonObject = null; + public JsonNode parseConfigFile(String configFile) { + JsonNode jsonNode = null; try{ - Object obj = parser.parse(new FileReader(configFile)); - jsonObject = (JSONObject) obj; + jsonNode = RelaxedJsonMapper.parseJson(new FileReader(configFile)); } catch (IOException e) { // Print error/abort/skip - } catch (ParseException e) { - // throw error/abort test/skip/test } - return jsonObject; + return jsonNode; + } + + // Helper method to convert JsonNode to HashMap + private HashMap getMapFromJsonNode(JsonNode node, String key) { + JsonNode childNode = node.get(key); + if (childNode != null) { + Map map = (Map) RelaxedJsonMapper.jsonNodeToObject(childNode); + HashMap result = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + result.put(entry.getKey(), entry.getValue().toString()); + } + return result; + } + return new HashMap<>(); } // String type data validation @@ -129,7 +138,7 @@ public void testValidateString() throws Exception { } // Create datafile - HashMap binMap = (HashMap) testSchema.get("test_string"); + HashMap binMap = getMapFromJsonNode(testSchema, "test_string"); int setMod = 5, range = 100, seed = 10, nrecords = 10; @@ -161,7 +170,7 @@ public void testValidateInteger() throws Exception { // Create datafile - HashMap binMap = (HashMap) testSchema.get("test_integer"); + HashMap binMap = getMapFromJsonNode(testSchema, "test_integer"); int setMod = 5, range = 100, seed = 10, nrecords = 10; @@ -193,7 +202,7 @@ public void testValidateStringUtf8() throws Exception { // Create datafile - HashMap binMap = (HashMap) testSchema.get("test_utf8"); + HashMap binMap = getMapFromJsonNode(testSchema, "test_utf8"); int setMod = 5, range = 100, seed = 10, nrecords = 10; @@ -224,7 +233,7 @@ public void testValidateTimestampInteger() throws Exception { } // Create datafile - HashMap binMap = (HashMap) testSchema.get("test_date"); + HashMap binMap = getMapFromJsonNode(testSchema, "test_date"); int setMod = 5, range = 100, seed = 10, nrecords = 10; @@ -255,7 +264,7 @@ public void testValidateBlob() throws Exception { } // Create datafile - HashMap binMap = (HashMap) testSchema.get("test_blob"); + HashMap binMap = getMapFromJsonNode(testSchema, "test_blob"); int setMod = 5, range = 100, seed = 10, nrecords = 10; @@ -286,7 +295,7 @@ public void testValidateList() throws Exception { } // Create datafile - HashMap binMap = (HashMap) testSchema.get("test_list"); + HashMap binMap = getMapFromJsonNode(testSchema, "test_list"); int setMod = 5, range = 100, seed = 10, nrecords = 10; @@ -318,7 +327,7 @@ public void testValidateMap() throws Exception { // Create datafile - HashMap binMap = (HashMap) testSchema.get("test_map"); + HashMap binMap = getMapFromJsonNode(testSchema, "test_map"); int setMod = 5, range = 100, seed = 10, nrecords = 10; @@ -350,7 +359,7 @@ public void testValidateJSON() throws Exception { // Create datafile - HashMap binMap = (HashMap) testSchema.get("test_json"); + HashMap binMap = getMapFromJsonNode(testSchema, "test_json"); int setMod = 5, range = 100, seed = 10, nrecords = 10; @@ -382,7 +391,7 @@ public void testAllDatatype() throws Exception { // Create datafile - HashMap binMap = (HashMap) testSchema.get("test_alltype"); + HashMap binMap = getMapFromJsonNode(testSchema, "test_alltype"); int setMod = 5, range = 100, seed = 10, nrecords = 10; @@ -446,7 +455,7 @@ public void testValidateMapOrder() throws Exception { // Create datafile - HashMap binMap = (HashMap) testSchema.get("test_map"); + HashMap binMap = getMapFromJsonNode(testSchema, "test_map"); int setMod = 5, range = 100, seed = 10, nrecords = 10; @@ -472,7 +481,7 @@ public void testValidateMapOrder() throws Exception { // Helper functions public void writeDataMap(String fileName, int nrecords, int setMod, int range, int seed, HashMap binMap) { - String delimiter = (String) testSchema.get("delimiter"); + String delimiter = testSchema.get("delimiter").asText(); File file = new File(fileName); // if file doesnt exists, then create it @@ -530,7 +539,7 @@ public boolean validateMap(AerospikeClient client, String filename, int nrecords boolean valid = false; Random r = new Random(seed); int rint; - String as_binname_suffix = (String) testSchema.get("as_binname_suffix"); + String as_binname_suffix = testSchema.get("as_binname_suffix").asText(); for (int i = 1; i <= nrecords; i++) { @@ -720,7 +729,6 @@ public String getValue(String binName, String binType, int i) { value = String.format("%d", i); break; case JSON: - JSONParser jsonParser = new JSONParser(); value = "{\"k1\": \"v1\", \"k2\": [\"lv1\", \"lv2\"], \"k3\": {\"mk1\": \"mv1\"}}"; break; case LIST: diff --git a/src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java b/src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java new file mode 100644 index 0000000..5294953 --- /dev/null +++ b/src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java @@ -0,0 +1,87 @@ +package com.aerospike.load; + +import org.junit.Test; +import static org.junit.Assert.*; + +import com.fasterxml.jackson.databind.JsonNode; +import java.util.Map; +import java.util.List; + +public class RelaxedJsonMapperTest { + + @Test + public void testParseJsonWithUnquotedKeys() throws Exception { + String json = "{name: 'John', age: 30}"; + JsonNode node = RelaxedJsonMapper.parseJson(json); + + assertNotNull(node); + assertTrue(node.isObject()); + assertEquals("John", node.get("name").asText()); + assertEquals(30, node.get("age").asInt()); + } + + @Test + public void testParseJsonWithKeyCoercion() throws Exception { + String json = "{1: 'value1', 2: 'value2', true: 'flag', '3.14': 'pi'}"; + Map result = RelaxedJsonMapper.parseJsonWithKeyCoercion(json); + + assertEquals("value1", result.get(1)); + assertEquals("value2", result.get(2)); + assertEquals("flag", result.get(true)); + assertEquals("pi", result.get(3.14)); + } + + @Test + public void testParseJsonToList() throws Exception { + String json = "[1, 2, 'three', true]"; + List list = RelaxedJsonMapper.parseJsonToList(json); + + assertEquals(4, list.size()); + assertEquals(1, list.get(0)); + assertEquals(2, list.get(1)); + assertEquals("three", list.get(2)); + assertEquals(true, list.get(3)); + } + + @Test + public void testParseJsonToMap() throws Exception { + String json = "{key1: 'value1', key2: 42, key3: true}"; + Map map = RelaxedJsonMapper.parseJsonToMap(json); + + assertEquals("value1", map.get("key1")); + assertEquals(42, map.get("key2")); + assertEquals(true, map.get("key3")); + } + + @Test + public void testGetFromJsonNode() throws Exception { + String json = "{name: 'Test', count: 5}"; + JsonNode node = RelaxedJsonMapper.parseJson(json); + + assertEquals("Test", RelaxedJsonMapper.getFromJsonNode(node, "name")); + assertEquals(5, RelaxedJsonMapper.getFromJsonNode(node, "count")); + assertNull(RelaxedJsonMapper.getFromJsonNode(node, "nonexistent")); + } + + @Test + public void testHasField() throws Exception { + String json = "{name: 'Test', count: 5}"; + JsonNode node = RelaxedJsonMapper.parseJson(json); + + assertTrue(RelaxedJsonMapper.hasField(node, "name")); + assertTrue(RelaxedJsonMapper.hasField(node, "count")); + assertFalse(RelaxedJsonMapper.hasField(node, "nonexistent")); + } + + @Test + public void testJsonNodeToObject() throws Exception { + String json = "{nested: {value: 123}, array: [1, 2, 3]}"; + JsonNode node = RelaxedJsonMapper.parseJson(json); + + Object nestedObj = RelaxedJsonMapper.jsonNodeToObject(node.get("nested")); + assertTrue(nestedObj instanceof Map); + + Object arrayObj = RelaxedJsonMapper.jsonNodeToObject(node.get("array")); + assertTrue(arrayObj instanceof List); + } +} \ No newline at end of file From 3df25707d079baf1c256061d2e8680988e580404 Mon Sep 17 00:00:00 2001 From: Rrutum <71127900+rrutum@users.noreply.github.com> Date: Tue, 15 Jul 2025 14:29:15 +0530 Subject: [PATCH 2/9] Fallback to relaxed json parsing if strict json parsing fails --- .../java/com/aerospike/load/AsWriterTask.java | 1185 +++++++++-------- 1 file changed, 597 insertions(+), 588 deletions(-) diff --git a/src/main/java/com/aerospike/load/AsWriterTask.java b/src/main/java/com/aerospike/load/AsWriterTask.java index ad75d87..1319bd6 100644 --- a/src/main/java/com/aerospike/load/AsWriterTask.java +++ b/src/main/java/com/aerospike/load/AsWriterTask.java @@ -20,591 +20,600 @@ * IN THE SOFTWARE. ******************************************************************************/ -package com.aerospike.load; - -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.TreeMap; -import java.util.concurrent.Callable; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import com.fasterxml.jackson.databind.JsonNode; -import java.io.IOException; -import java.util.Map; - -import com.aerospike.client.AerospikeClient; -import com.aerospike.client.AerospikeException; -import com.aerospike.client.Bin; -import com.aerospike.client.Key; -import com.aerospike.client.Value; -import com.aerospike.client.ResultCode; - -/** - * - * @author Aerospike - * - * Main writer class to write data in Aerospike. - * - */ -public class AsWriterTask implements Callable { - - // File and line info variable. - private String fileName; - private int lineNumber; - private int lineSize; - - // Aerospike related variable. - private AerospikeClient client; - - // Data definition related variable - private HashMap dsvConfigs; - private MappingDefinition mappingDef; - private List columns; - - private Parameters params; - private Counter counters; - // JSON parsing is now handled by RelaxedJsonMapper - - private static Logger log = LogManager.getLogger(AsWriterTask.class); - - /** - * AsWriterTask process given data columns for a record and create Set and Key and Bins. - * It writes these Bins to created Key. If its secondary mapping then it will do CDT append - * over all created Bins. - * - * @param fileName Name of the data file - * @param lineNumber Line number in the file fileName - * @param lineSize Size of the line to keep track of record processed - * @param client AerospikeClient object - * @param columns List of column separated entries in this lineNumber - * @param dsvConfig Map of DSV configurations - * @param mappingDef MappingDefinition of a mapping from config file - * @param params User given parameters - * @param counters Counter for stats - * - */ - public AsWriterTask(String fileName, int lineNumber, int lineSize,AerospikeClient client, List columns, - HashMap dsvConfigs, MappingDefinition mappingDef, Parameters params, Counter counters) { - - // Passed to print log error with filename, line number, increment byteprocessed. - this.fileName = fileName; - this.lineNumber = lineNumber; - this.lineSize = lineSize; - - this.client = client; - - this.dsvConfigs = dsvConfigs; - this.mappingDef = mappingDef; - this.columns = columns; - - this.params = params; - this.counters = counters; - - } - - /* - * Writes a record to the Aerospike Cluster - */ - private void writeToAs(Key key, List bins) { - - try { - // Connection could be broken at actual write time. - if (this.client == null) { - throw new Exception("Null Aerospike client !!"); - } - - - if (bins.isEmpty()) { - counters.write.noBinsSkipped.getAndIncrement(); - log.trace("No bins to insert"); - return; - } - // All bins will have append operation if secondary mapping. - if (this.mappingDef.secondaryMapping) { - for (Bin b : bins) { - client.operate(this.params.writePolicy, key, - com.aerospike.client.cdt.ListOperation.append(b.name, b.value)); - } - counters.write.mappingWriteCount.getAndIncrement(); - } else { - this.client.put(this.params.writePolicy, key, bins.toArray(new Bin[bins.size()])); - counters.write.bytesProcessed.addAndGet(this.lineSize); - } - counters.write.writeCount.getAndIncrement(); - - log.trace("Wrote line " + lineNumber + " Key: " + key.userKey + " to Aerospike."); - - } catch (AerospikeException ae) { - - handleAerospikeWriteError(ae); - checkAndAbort(); - - } catch (Exception e) { - - handleWriteError(e); - checkAndAbort(); - - } - } - - private void handleAerospikeWriteError(AerospikeException ae) { - - log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + "Aerospike Write Error: " - + ae.getResultCode()); - - if (log.isDebugEnabled()) { - ae.printStackTrace(); - } - - switch (ae.getResultCode()) { - - case ResultCode.TIMEOUT: - counters.write.writeTimeouts.getAndIncrement(); - break; - case ResultCode.KEY_EXISTS_ERROR: - counters.write.writeKeyExists.getAndIncrement(); - break; - default: - //.. - } - - if (!this.mappingDef.secondaryMapping) { - counters.write.bytesProcessed.addAndGet(this.lineSize); - } - - counters.write.writeErrors.getAndIncrement(); - } - - private void handleWriteError(Exception e) { - log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + " Write Error: " + e); - if (log.isDebugEnabled()) { - e.printStackTrace(); - } - if (!this.mappingDef.secondaryMapping) { - counters.write.bytesProcessed.addAndGet(this.lineSize); - } - counters.write.writeErrors.getAndIncrement(); - } - - private void checkAndAbort(){ - long errorTotal; - errorTotal = (counters.write.readErrors.get() + counters.write.writeErrors.get() - + counters.write.processingErrors.get()); - if (this.params.abortErrorCount != 0 && this.params.abortErrorCount < errorTotal) { - System.exit(-1); - } - } - - /* - * Create Set and Key from provided data for given mappingDef. - * Create Bin for given binList in mappingDef. - */ - private Key getKeyAndBinsFromDataline(List bins) { - log.debug("processing File: " + Utils.getFileName(fileName) + "line: " + this.lineNumber); - Key key = null; - - try { - - validateNColumnInDataline(); - - // Set couldn't be null here. Its been validated earlier. - String set = getSetName(); - - key = createRecordKey(this.params.namespace, set); - - populateAsBinFromColumnDef(bins); - - log.trace("Formed key and bins for line: " + lineNumber + " Key: " + key.userKey + " Bins: " - + bins.toString()); - - } catch (AerospikeException ae) { - - log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber - + " Aerospike Bin processing Error: " + ae.getResultCode()); - handleProcessLineError(ae); - - } catch (Exception e) { - - log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + " Error: " + e); - handleProcessLineError(e); - - } - return key; - } - - /* - * Validate if number of column in data line are same as provided in config file. - * Throw exception the more columns are present then given. - */ - private void validateNColumnInDataline() throws Exception { - - // Throw exception if n_columns(datafile) are more than n_columns(configfile). - int n_column = Integer.parseInt(dsvConfigs.get(Constants.N_COLUMN)); - if (columns.size() == n_column) { - return; - } - if (columns.size() < n_column) { - log.warn("File: " + Utils.getFileName(fileName) + " Line: " + lineNumber - + " Number of column mismatch:Columns in data file is less than number of column in config file."); - } else { - throw new Exception("Column count mismatch at line " + lineNumber); - } - } - - private void handleProcessLineError(Exception e) { - if (log.isDebugEnabled()) { - e.printStackTrace(); - } - counters.write.processingErrors.getAndIncrement(); - counters.write.bytesProcessed.addAndGet(this.lineSize); - checkAndAbort(); - } - - - private String getSetName() { - MetaDefinition setColumn = this.mappingDef.setColumnDef; - - if (setColumn.staticName != null) { - return setColumn.staticName; - } - - String set = null; - String setRawText = this.columns.get(setColumn.nameDef.columnPos); - if (setColumn.nameDef.removePrefix != null) { - if (setRawText != null && setRawText.startsWith(setColumn.nameDef.removePrefix)) { - set = setRawText.substring(setColumn.nameDef.removePrefix.length()); - } - } else { - set = setRawText; - } - return set; - } - - private Key createRecordKey(String namespace, String set) throws Exception { - // Use 'SET' name to create key. - Key key = null; - - MetaDefinition keyColumn = this.mappingDef.keyColumnDef; - - String keyRawText = this.columns.get(keyColumn.nameDef.columnPos); - - if (keyRawText == null || keyRawText.trim().length() == 0) { - counters.write.keyNullSkipped.getAndIncrement(); - throw new Exception("Key value is null in datafile."); - } - - if ((keyColumn.nameDef.removePrefix != null) - && (keyRawText.startsWith(keyColumn.nameDef.removePrefix))) { - keyRawText = keyRawText.substring(keyColumn.nameDef.removePrefix.length()); - } - - if (keyColumn.nameDef.srcType == SrcColumnType.INTEGER) { - Long integer = Long.parseLong(keyRawText); - key = new Key(namespace, set, integer); - } else { - key = new Key(namespace, set, keyRawText); - } - - return key; - } - - private void populateAsBinFromColumnDef(List bins) { - for (BinDefinition binColumn : this.mappingDef.binColumnDefs) { - Bin bin = null; - String binName = null; - String binRawValue = null; - - // Get binName. - if (binColumn.staticName != null) { - binName = binColumn.staticName; - } else if (binColumn.nameDef != null) { - binName = this.columns.get(binColumn.nameDef.columnPos); - } - - // Get BinValue. - if (binColumn.staticValue != null) { - - binRawValue = binColumn.staticValue; - bin = new Bin(binName, binRawValue); - - } else if (binColumn.valueDef != null) { - - binRawValue = getbinRawValue(binColumn); - if (binRawValue == null || binRawValue.equals("")) { - continue; - } - - switch (binColumn.valueDef.srcType) { - - case INTEGER: - bin = createBinForInteger(binName, binRawValue); - break; - case FLOAT: - bin = createBinForFloat(binName, binRawValue); - break; - case STRING: - bin = createBinForString(binName, binRawValue); - break; - case JSON: - /* - * JSON could take any valid JSON. There are two type of JSON: - * JsonArray: this can be used to insert List (Any generic JSON list) - * JsonObj: this can be used to insert Map (Any generic JSON object) - */ - bin = createBinForJson(binName, binRawValue); - break; - case GEOJSON: - bin = createBinForGeoJson(binName, binRawValue); - break; - case BLOB: - bin = createBinForBlob(binColumn, binName, binRawValue); - break; - case TIMESTAMP: - bin = createBinForTimestamp(binColumn, binName, binRawValue); - break; - default: - //.... - } - } - - if (bin != null) { - bins.add(bin); - } - } - } - - private String getbinRawValue(BinDefinition binColumn) { - /* - * User may want to store the time when record is written in Aerospike. - * Assign system_time to binvalue. This bin will be written as part of - * record. - */ - if (binColumn.valueDef.columnName != null - && binColumn.valueDef.columnName.toLowerCase().equals(Constants.SYSTEM_TIME)) { - - SimpleDateFormat sdf = - new SimpleDateFormat(binColumn.valueDef.encoding); // dd/MM/yyyy - Date now = new Date(); - return sdf.format(now); - } - - - String binRawValue = this.columns.get(binColumn.valueDef.columnPos); - - if ((binColumn.valueDef.removePrefix != null) - && (binRawValue != null && binRawValue.startsWith(binColumn.valueDef.removePrefix))) { - binRawValue = - binRawValue.substring(binColumn.valueDef.removePrefix.length()); - } - return binRawValue; - } - - private Bin createBinForInteger(String binName, String binRawValue) { - - try { - // Server stores all integer type data in 64bit so use long. - Long integer = Long.parseLong(binRawValue); - - return new Bin(binName, integer); - - } catch (Exception pi) { - - log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber - + " Integer/Long Parse Error: " + pi); - return null; - - } - } - - private Bin createBinForFloat(String binName, String binRawValue) { - - try { - // parse as a double to get greater precision - double binfloat = Double.parseDouble(binRawValue); - - return new Bin(binName, binfloat); - - } catch (Exception e) { - log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber - + " Floating number Parse Error: " + e); - return null; - } - - } - - private Bin createBinForString(String binName, String binRawValue) { - return new Bin(binName, binRawValue); - } - - private Bin createBinForJson(String binName, String binRawValue) { - try { - log.debug(binRawValue); - - JsonNode jsonNode = RelaxedJsonMapper.parseJson(binRawValue); - - if (jsonNode.isArray()) { - List jsonArray = RelaxedJsonMapper.parseJsonToList(binRawValue); - return new Bin(binName, jsonArray); - } else { - Object jsonObj = RelaxedJsonMapper.jsonNodeToObject(jsonNode); - - if (this.params.unorderdMaps) { - if (jsonObj instanceof Map) { - return new Bin(binName, (Map) jsonObj); - } - return new Bin(binName, jsonObj.toString()); - } - - if (jsonObj instanceof Map) { - TreeMap sortedMap = new TreeMap<>(); - sortedMap.putAll((Map) jsonObj); - return new Bin(binName, sortedMap); - } - - return new Bin(binName, jsonObj.toString()); - } - - } catch (IOException e) { - log.error("Failed to parse JSON: " + e); - return null; - } - - } - - private Bin createBinForGeoJson(String binName, String binRawValue) { - try { - return new Bin(binName, Value.getAsGeoJSON(binRawValue)); - } catch (Exception e) { - log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber - + " GeoJson Parse Error: " + e); - return null; - } - } - - private Bin createBinForBlob(BinDefinition binColumn, String binName, String binRawValue) { - try { - if ((binColumn.valueDef.dstType.equals(DstColumnType.BLOB)) - && (binColumn.valueDef.encoding.equalsIgnoreCase(Constants.HEX_ENCODING))) { - return new Bin(binName, this.toByteArray(binRawValue)); - } - } catch (Exception e) { - log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber - + " Blob Parse Error: " + e); - return null; - } - - return null; - } - - private Bin createBinForTimestamp(BinDefinition binColumn, String binName, String binRawValue) { - - if (! binColumn.valueDef.dstType.equals(DstColumnType.INTEGER)) { - return new Bin(binName, binRawValue); - } - - DateFormat format = new SimpleDateFormat(binColumn.valueDef.encoding); - - try { - - Date formatDate = format.parse(binRawValue); - long milliSecondForDate = formatDate.getTime() - this.params.timeZoneOffset; - - if (!(binColumn.valueDef.encoding.contains(".SSS") - && binName.equals(Constants.SYSTEM_TIME))) { - // We need time in milliseconds so no need to change it to milliseconds. - milliSecondForDate = milliSecondForDate / 1000; - } - - log.trace("Date format: " + binRawValue + " in seconds: " + milliSecondForDate); - - return new Bin(binName, milliSecondForDate); - - } catch (java.text.ParseException e) { - e.printStackTrace(); - return null; - } - - } - - private byte[] toByteArray(String s) { - - if ((s.length() % 2) != 0) { - log.error("blob exception: " + s); - throw new IllegalArgumentException("Input hex formated string must contain an even number of characters."); - } - - int len = s.length(); - byte[] data = new byte[len / 2]; - - try { - for (int i = 0; i < len; i += 2) { - data[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) << 4) + Character.digit(s.charAt(i + 1), 16)); - } - } catch (Exception e) { - log.error("blob exception: " + e); - } - return data; - } - - private boolean exceedingThroughput() { - long transactions; - long timeLapse; - long throughput; - - transactions = counters.write.writeCount.get() - + counters.write.mappingWriteCount.get() - + counters.write.writeErrors.get(); - - - timeLapse = (System.currentTimeMillis() - counters.write.writeStartTime) / 1000L; - - if (timeLapse > 0) { - throughput = transactions / timeLapse; - - if (throughput > params.maxThroughput) { - return true; - } - } - return false; - } - public Integer call() throws Exception { - - List bins = new ArrayList(); - - - try { - - counters.write.processingCount.getAndIncrement(); - - Key key = getKeyAndBinsFromDataline(bins); - - if (key != null) { - writeToAs(key, bins); - bins.clear(); - - if (params.maxThroughput == 0) { - return 0; - } - - while(exceedingThroughput()) { - Thread.sleep(20); - } - - return 0; - } - - } catch (Exception e) { - - log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + " Parsing Error: " + e); - log.debug(e); - } - - return 0; - - } -} + package com.aerospike.load; + + import java.text.DateFormat; + import java.text.SimpleDateFormat; + import java.util.ArrayList; + import java.util.Date; + import java.util.HashMap; + import java.util.List; + import java.util.TreeMap; + import java.util.concurrent.Callable; + + import org.apache.logging.log4j.LogManager; + import org.apache.logging.log4j.Logger; + import java.io.IOException; + import java.util.Map; + import com.fasterxml.jackson.databind.ObjectMapper; + import com.fasterxml.jackson.core.JsonProcessingException; + + import com.aerospike.client.AerospikeClient; + import com.aerospike.client.AerospikeException; + import com.aerospike.client.Bin; + import com.aerospike.client.Key; + import com.aerospike.client.Value; + import com.aerospike.client.ResultCode; + + /** + * + * @author Aerospike + * + * Main writer class to write data in Aerospike. + * + */ + public class AsWriterTask implements Callable { + + // File and line info variable. + private String fileName; + private int lineNumber; + private int lineSize; + + // Aerospike related variable. + private AerospikeClient client; + + // Data definition related variable + private HashMap dsvConfigs; + private MappingDefinition mappingDef; + private List columns; + + private Parameters params; + private Counter counters; + // JSON parsing is now handled by RelaxedJsonMapper + + private static Logger log = LogManager.getLogger(AsWriterTask.class); + + /** + * AsWriterTask process given data columns for a record and create Set and Key and Bins. + * It writes these Bins to created Key. If its secondary mapping then it will do CDT append + * over all created Bins. + * + * @param fileName Name of the data file + * @param lineNumber Line number in the file fileName + * @param lineSize Size of the line to keep track of record processed + * @param client AerospikeClient object + * @param columns List of column separated entries in this lineNumber + * @param dsvConfigs Map of DSV configurations + * @param mappingDef MappingDefinition of a mapping from config file + * @param params User given parameters + * @param counters Counter for stats + * + */ + public AsWriterTask(String fileName, int lineNumber, int lineSize,AerospikeClient client, List columns, + HashMap dsvConfigs, MappingDefinition mappingDef, Parameters params, Counter counters) { + + // Passed to print log error with filename, line number, increment byteprocessed. + this.fileName = fileName; + this.lineNumber = lineNumber; + this.lineSize = lineSize; + + this.client = client; + + this.dsvConfigs = dsvConfigs; + this.mappingDef = mappingDef; + this.columns = columns; + + this.params = params; + this.counters = counters; + + } + + /* + * Writes a record to the Aerospike Cluster + */ + private void writeToAs(Key key, List bins) { + + try { + // Connection could be broken at actual write time. + if (this.client == null) { + throw new Exception("Null Aerospike client !!"); + } + + + if (bins.isEmpty()) { + counters.write.noBinsSkipped.getAndIncrement(); + log.trace("No bins to insert"); + return; + } + // All bins will have append operation if secondary mapping. + if (this.mappingDef.secondaryMapping) { + for (Bin b : bins) { + client.operate(this.params.writePolicy, key, + com.aerospike.client.cdt.ListOperation.append(b.name, b.value)); + } + counters.write.mappingWriteCount.getAndIncrement(); + } else { + this.client.put(this.params.writePolicy, key, bins.toArray(new Bin[bins.size()])); + counters.write.bytesProcessed.addAndGet(this.lineSize); + } + counters.write.writeCount.getAndIncrement(); + + log.trace("Wrote line " + lineNumber + " Key: " + key.userKey + " to Aerospike."); + + } catch (AerospikeException ae) { + + handleAerospikeWriteError(ae); + checkAndAbort(); + + } catch (Exception e) { + + handleWriteError(e); + checkAndAbort(); + + } + } + + private void handleAerospikeWriteError(AerospikeException ae) { + + log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + "Aerospike Write Error: " + + ae.getResultCode()); + + if (log.isDebugEnabled()) { + ae.printStackTrace(); + } + + switch (ae.getResultCode()) { + + case ResultCode.TIMEOUT: + counters.write.writeTimeouts.getAndIncrement(); + break; + case ResultCode.KEY_EXISTS_ERROR: + counters.write.writeKeyExists.getAndIncrement(); + break; + default: + //.. + } + + if (!this.mappingDef.secondaryMapping) { + counters.write.bytesProcessed.addAndGet(this.lineSize); + } + + counters.write.writeErrors.getAndIncrement(); + } + + private void handleWriteError(Exception e) { + log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + " Write Error: " + e); + if (log.isDebugEnabled()) { + e.printStackTrace(); + } + if (!this.mappingDef.secondaryMapping) { + counters.write.bytesProcessed.addAndGet(this.lineSize); + } + counters.write.writeErrors.getAndIncrement(); + } + + private void checkAndAbort(){ + long errorTotal; + errorTotal = (counters.write.readErrors.get() + counters.write.writeErrors.get() + + counters.write.processingErrors.get()); + if (this.params.abortErrorCount != 0 && this.params.abortErrorCount < errorTotal) { + System.exit(-1); + } + } + + /* + * Create Set and Key from provided data for given mappingDef. + * Create Bin for given binList in mappingDef. + */ + private Key getKeyAndBinsFromDataline(List bins) { + log.debug("processing File: " + Utils.getFileName(fileName) + "line: " + this.lineNumber); + Key key = null; + + try { + + validateNColumnInDataline(); + + // Set couldn't be null here. Its been validated earlier. + String set = getSetName(); + + key = createRecordKey(this.params.namespace, set); + + populateAsBinFromColumnDef(bins); + + log.trace("Formed key and bins for line: " + lineNumber + " Key: " + key.userKey + " Bins: " + + bins.toString()); + + } catch (AerospikeException ae) { + + log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + + " Aerospike Bin processing Error: " + ae.getResultCode()); + handleProcessLineError(ae); + + } catch (Exception e) { + + log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + " Error: " + e); + handleProcessLineError(e); + + } + return key; + } + + /* + * Validate if number of column in data line are same as provided in config file. + * Throw exception the more columns are present then given. + */ + private void validateNColumnInDataline() throws Exception { + + // Throw exception if n_columns(datafile) are more than n_columns(configfile). + int n_column = Integer.parseInt(dsvConfigs.get(Constants.N_COLUMN)); + if (columns.size() == n_column) { + return; + } + if (columns.size() < n_column) { + log.warn("File: " + Utils.getFileName(fileName) + " Line: " + lineNumber + + " Number of column mismatch:Columns in data file is less than number of column in config file."); + } else { + throw new Exception("Column count mismatch at line " + lineNumber); + } + } + + private void handleProcessLineError(Exception e) { + if (log.isDebugEnabled()) { + e.printStackTrace(); + } + counters.write.processingErrors.getAndIncrement(); + counters.write.bytesProcessed.addAndGet(this.lineSize); + checkAndAbort(); + } + + + private String getSetName() { + MetaDefinition setColumn = this.mappingDef.setColumnDef; + + if (setColumn.staticName != null) { + return setColumn.staticName; + } + + String set = null; + String setRawText = this.columns.get(setColumn.nameDef.columnPos); + if (setColumn.nameDef.removePrefix != null) { + if (setRawText != null && setRawText.startsWith(setColumn.nameDef.removePrefix)) { + set = setRawText.substring(setColumn.nameDef.removePrefix.length()); + } + } else { + set = setRawText; + } + return set; + } + + private Key createRecordKey(String namespace, String set) throws Exception { + // Use 'SET' name to create key. + Key key = null; + + MetaDefinition keyColumn = this.mappingDef.keyColumnDef; + + String keyRawText = this.columns.get(keyColumn.nameDef.columnPos); + + if (keyRawText == null || keyRawText.trim().length() == 0) { + counters.write.keyNullSkipped.getAndIncrement(); + throw new Exception("Key value is null in datafile."); + } + + if ((keyColumn.nameDef.removePrefix != null) + && (keyRawText.startsWith(keyColumn.nameDef.removePrefix))) { + keyRawText = keyRawText.substring(keyColumn.nameDef.removePrefix.length()); + } + + if (keyColumn.nameDef.srcType == SrcColumnType.INTEGER) { + Long integer = Long.parseLong(keyRawText); + key = new Key(namespace, set, integer); + } else { + key = new Key(namespace, set, keyRawText); + } + + return key; + } + + private void populateAsBinFromColumnDef(List bins) { + for (BinDefinition binColumn : this.mappingDef.binColumnDefs) { + Bin bin = null; + String binName = null; + String binRawValue = null; + + // Get binName. + if (binColumn.staticName != null) { + binName = binColumn.staticName; + } else if (binColumn.nameDef != null) { + binName = this.columns.get(binColumn.nameDef.columnPos); + } + + // Get BinValue. + if (binColumn.staticValue != null) { + + binRawValue = binColumn.staticValue; + bin = new Bin(binName, binRawValue); + + } else if (binColumn.valueDef != null) { + + binRawValue = getbinRawValue(binColumn); + if (binRawValue == null || binRawValue.equals("")) { + continue; + } + + switch (binColumn.valueDef.srcType) { + + case INTEGER: + bin = createBinForInteger(binName, binRawValue); + break; + case FLOAT: + bin = createBinForFloat(binName, binRawValue); + break; + case STRING: + bin = createBinForString(binName, binRawValue); + break; + case JSON: + /* + * JSON could take any valid JSON. There are two type of JSON: + * JsonArray: this can be used to insert List (Any generic JSON list) + * JsonObj: this can be used to insert Map (Any generic JSON object) + */ + bin = createBinForJson(binName, binRawValue); + break; + case GEOJSON: + bin = createBinForGeoJson(binName, binRawValue); + break; + case BLOB: + bin = createBinForBlob(binColumn, binName, binRawValue); + break; + case TIMESTAMP: + bin = createBinForTimestamp(binColumn, binName, binRawValue); + break; + default: + //.... + } + } + + if (bin != null) { + bins.add(bin); + } + } + } + + private String getbinRawValue(BinDefinition binColumn) { + /* + * User may want to store the time when record is written in Aerospike. + * Assign system_time to binvalue. This bin will be written as part of + * record. + */ + if (binColumn.valueDef.columnName != null + && binColumn.valueDef.columnName.toLowerCase().equals(Constants.SYSTEM_TIME)) { + + SimpleDateFormat sdf = + new SimpleDateFormat(binColumn.valueDef.encoding); // dd/MM/yyyy + Date now = new Date(); + return sdf.format(now); + } + + + String binRawValue = this.columns.get(binColumn.valueDef.columnPos); + + if ((binColumn.valueDef.removePrefix != null) + && (binRawValue != null && binRawValue.startsWith(binColumn.valueDef.removePrefix))) { + binRawValue = + binRawValue.substring(binColumn.valueDef.removePrefix.length()); + } + return binRawValue; + } + + private Bin createBinForInteger(String binName, String binRawValue) { + + try { + // Server stores all integer type data in 64bit so use long. + Long integer = Long.parseLong(binRawValue); + + return new Bin(binName, integer); + + } catch (Exception pi) { + + log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + + " Integer/Long Parse Error: " + pi); + return null; + + } + } + + private Bin createBinForFloat(String binName, String binRawValue) { + + try { + // parse as a double to get greater precision + double binfloat = Double.parseDouble(binRawValue); + + return new Bin(binName, binfloat); + + } catch (Exception e) { + log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + + " Floating number Parse Error: " + e); + return null; + } + + } + + private Bin createBinForString(String binName, String binRawValue) { + return new Bin(binName, binRawValue); + } + + private Bin createBinForJson(String binName, String binRawValue) { + try { + log.debug(binRawValue); + + try { + ObjectMapper standardMapper = new ObjectMapper(); + Object obj = standardMapper.readValue(binRawValue, Object.class); + + if (obj instanceof List) { + List jsonArray = (List) obj; + return new Bin(binName, jsonArray); + } else if (obj instanceof Map) { + Map jsonObj = (Map) obj; + + if (this.params.unorderdMaps) { + return new Bin(binName, jsonObj); + } + + try { + TreeMap sortedMap = new TreeMap<>(); + sortedMap.putAll(jsonObj); + return new Bin(binName, sortedMap); + } catch (ClassCastException e) { + // Keys not comparable, fall back to unordered map + log.debug("TreeMap failed due to non-comparable keys, using unordered map: " + e.getMessage()); + return new Bin(binName, jsonObj); + } + } else { + return new Bin(binName, obj.toString()); + } + } catch (JsonProcessingException standardParseException) { + log.debug("Standard JSON parsing failed, using relaxed parser: " + standardParseException.getMessage()); + Map relaxedResult = RelaxedJsonMapper.parseJsonWithKeyCoercion(binRawValue); + return new Bin(binName, relaxedResult); + } + + } catch (IOException e) { + log.error("Failed to parse JSON: " + e); + return null; + } + } + + private Bin createBinForGeoJson(String binName, String binRawValue) { + try { + return new Bin(binName, Value.getAsGeoJSON(binRawValue)); + } catch (Exception e) { + log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + + " GeoJson Parse Error: " + e); + return null; + } + } + + private Bin createBinForBlob(BinDefinition binColumn, String binName, String binRawValue) { + try { + if ((binColumn.valueDef.dstType.equals(DstColumnType.BLOB)) + && (binColumn.valueDef.encoding.equalsIgnoreCase(Constants.HEX_ENCODING))) { + return new Bin(binName, this.toByteArray(binRawValue)); + } + } catch (Exception e) { + log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + + " Blob Parse Error: " + e); + return null; + } + + return null; + } + + private Bin createBinForTimestamp(BinDefinition binColumn, String binName, String binRawValue) { + + if (! binColumn.valueDef.dstType.equals(DstColumnType.INTEGER)) { + return new Bin(binName, binRawValue); + } + + DateFormat format = new SimpleDateFormat(binColumn.valueDef.encoding); + + try { + + Date formatDate = format.parse(binRawValue); + long milliSecondForDate = formatDate.getTime() - this.params.timeZoneOffset; + + if (!(binColumn.valueDef.encoding.contains(".SSS") + && binName.equals(Constants.SYSTEM_TIME))) { + // We need time in milliseconds so no need to change it to milliseconds. + milliSecondForDate = milliSecondForDate / 1000; + } + + log.trace("Date format: " + binRawValue + " in seconds: " + milliSecondForDate); + + return new Bin(binName, milliSecondForDate); + + } catch (java.text.ParseException e) { + e.printStackTrace(); + return null; + } + + } + + private byte[] toByteArray(String s) { + + if ((s.length() % 2) != 0) { + log.error("blob exception: " + s); + throw new IllegalArgumentException("Input hex formated string must contain an even number of characters."); + } + + int len = s.length(); + byte[] data = new byte[len / 2]; + + try { + for (int i = 0; i < len; i += 2) { + data[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) << 4) + Character.digit(s.charAt(i + 1), 16)); + } + } catch (Exception e) { + log.error("blob exception: " + e); + } + return data; + } + + private boolean exceedingThroughput() { + long transactions; + long timeLapse; + long throughput; + + transactions = counters.write.writeCount.get() + + counters.write.mappingWriteCount.get() + + counters.write.writeErrors.get(); + + + timeLapse = (System.currentTimeMillis() - counters.write.writeStartTime) / 1000L; + + if (timeLapse > 0) { + throughput = transactions / timeLapse; + + if (throughput > params.maxThroughput) { + return true; + } + } + return false; + } + public Integer call() throws Exception { + + List bins = new ArrayList(); + + + try { + + counters.write.processingCount.getAndIncrement(); + + Key key = getKeyAndBinsFromDataline(bins); + + if (key != null) { + writeToAs(key, bins); + bins.clear(); + + if (params.maxThroughput == 0) { + return 0; + } + + while(exceedingThroughput()) { + Thread.sleep(20); + } + + return 0; + } + + } catch (Exception e) { + + log.error("File: " + Utils.getFileName(this.fileName) + " Line: " + lineNumber + " Parsing Error: " + e); + log.debug(e); + } + + return 0; + + } + } + \ No newline at end of file From 0af588eae0267953f02020a2705f92b159179900 Mon Sep 17 00:00:00 2001 From: Rrutum <71127900+rrutum@users.noreply.github.com> Date: Tue, 15 Jul 2025 14:33:05 +0530 Subject: [PATCH 3/9] Fallback to relaxed json parsing if strict json parsing fails --- .../java/com/aerospike/load/AsWriterTask.java | 89 ++++++++++--------- 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/src/main/java/com/aerospike/load/AsWriterTask.java b/src/main/java/com/aerospike/load/AsWriterTask.java index 1319bd6..d00f0bd 100644 --- a/src/main/java/com/aerospike/load/AsWriterTask.java +++ b/src/main/java/com/aerospike/load/AsWriterTask.java @@ -83,7 +83,7 @@ public class AsWriterTask implements Callable { * @param lineSize Size of the line to keep track of record processed * @param client AerospikeClient object * @param columns List of column separated entries in this lineNumber - * @param dsvConfigs Map of DSV configurations + * @param dsvConfig Map of DSV configurations * @param mappingDef MappingDefinition of a mapping from config file * @param params User given parameters * @param counters Counter for stats @@ -443,47 +443,49 @@ private Bin createBinForString(String binName, String binRawValue) { } private Bin createBinForJson(String binName, String binRawValue) { - try { - log.debug(binRawValue); - - try { - ObjectMapper standardMapper = new ObjectMapper(); - Object obj = standardMapper.readValue(binRawValue, Object.class); - - if (obj instanceof List) { - List jsonArray = (List) obj; - return new Bin(binName, jsonArray); - } else if (obj instanceof Map) { - Map jsonObj = (Map) obj; - - if (this.params.unorderdMaps) { - return new Bin(binName, jsonObj); - } - - try { - TreeMap sortedMap = new TreeMap<>(); - sortedMap.putAll(jsonObj); - return new Bin(binName, sortedMap); - } catch (ClassCastException e) { - // Keys not comparable, fall back to unordered map - log.debug("TreeMap failed due to non-comparable keys, using unordered map: " + e.getMessage()); - return new Bin(binName, jsonObj); - } - } else { - return new Bin(binName, obj.toString()); - } - } catch (JsonProcessingException standardParseException) { - log.debug("Standard JSON parsing failed, using relaxed parser: " + standardParseException.getMessage()); - Map relaxedResult = RelaxedJsonMapper.parseJsonWithKeyCoercion(binRawValue); - return new Bin(binName, relaxedResult); - } - - } catch (IOException e) { - log.error("Failed to parse JSON: " + e); - return null; - } - } - + try { + log.debug(binRawValue); + + try { + ObjectMapper standardMapper = new ObjectMapper(); + Object obj = standardMapper.readValue(binRawValue, Object.class); + + if (obj instanceof List) { + List jsonArray = (List) obj; + return new Bin(binName, jsonArray); + } else if (obj instanceof Map) { + Map jsonObj = (Map) obj; + + if (this.params.unorderdMaps) { + return new Bin(binName, jsonObj); + } + + try { + TreeMap sortedMap = new TreeMap<>(); + sortedMap.putAll(jsonObj); + return new Bin(binName, sortedMap); + } catch (ClassCastException e) { + // Keys not comparable, fall back to unordered map + log.debug("TreeMap failed due to non-comparable keys, using unordered map: " + e.getMessage()); + return new Bin(binName, jsonObj); + } + } else { + return new Bin(binName, obj.toString()); + } + + } catch (JsonProcessingException standardParseException) { + // Standard JSON parsing failed, fall back to relaxed parser + log.debug("Standard JSON parsing failed, using relaxed parser: " + standardParseException.getMessage()); + Map relaxedResult = RelaxedJsonMapper.parseJsonWithKeyCoercion(binRawValue); + return new Bin(binName, relaxedResult); + } + + } catch (IOException e) { + log.error("Failed to parse JSON: " + e); + return null; + } + } + private Bin createBinForGeoJson(String binName, String binRawValue) { try { return new Bin(binName, Value.getAsGeoJSON(binRawValue)); @@ -615,5 +617,4 @@ public Integer call() throws Exception { return 0; } - } - \ No newline at end of file + } \ No newline at end of file From 29b4df2e98333bc98cc5633e3acd8e9742a164b5 Mon Sep 17 00:00:00 2001 From: Rrutum <71127900+rrutum@users.noreply.github.com> Date: Tue, 15 Jul 2025 14:35:26 +0530 Subject: [PATCH 4/9] Fix docstring --- src/main/java/com/aerospike/load/AsWriterTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/aerospike/load/AsWriterTask.java b/src/main/java/com/aerospike/load/AsWriterTask.java index d00f0bd..f883875 100644 --- a/src/main/java/com/aerospike/load/AsWriterTask.java +++ b/src/main/java/com/aerospike/load/AsWriterTask.java @@ -83,7 +83,7 @@ public class AsWriterTask implements Callable { * @param lineSize Size of the line to keep track of record processed * @param client AerospikeClient object * @param columns List of column separated entries in this lineNumber - * @param dsvConfig Map of DSV configurations + * @param dsvConfigs Map of DSV configurations * @param mappingDef MappingDefinition of a mapping from config file * @param params User given parameters * @param counters Counter for stats From 76bc2683aacf6a17e95792bfe26463908ce1b12f Mon Sep 17 00:00:00 2001 From: Rrutum <71127900+rrutum@users.noreply.github.com> Date: Tue, 15 Jul 2025 20:02:20 +0530 Subject: [PATCH 5/9] Convert to Map --- src/main/java/com/aerospike/load/Parser.java | 2 - .../com/aerospike/load/RelaxedJsonMapper.java | 40 +------------------ 2 files changed, 1 insertion(+), 41 deletions(-) diff --git a/src/main/java/com/aerospike/load/Parser.java b/src/main/java/com/aerospike/load/Parser.java index c8f28a4..6cec1d4 100644 --- a/src/main/java/com/aerospike/load/Parser.java +++ b/src/main/java/com/aerospike/load/Parser.java @@ -49,8 +49,6 @@ public class Parser { * @param configFile Config/schema/definition file name * @param dsvConfigs DSV configuration for loader to use, given in config file (version, n_columns, delimiter...) * @param mappingDefs List of schema/definitions for all mappings. primary + secondary mappings. - * @param params User given parameters - * @throws ParseException * @throws IOException */ public static boolean parseJSONColumnDefinitions(File configFile, HashMap dsvConfigs, List mappingDefs) { diff --git a/src/main/java/com/aerospike/load/RelaxedJsonMapper.java b/src/main/java/com/aerospike/load/RelaxedJsonMapper.java index b6f8db5..6e8f9d6 100644 --- a/src/main/java/com/aerospike/load/RelaxedJsonMapper.java +++ b/src/main/java/com/aerospike/load/RelaxedJsonMapper.java @@ -78,45 +78,7 @@ public static List parseJsonToList(String json) throws IOException { * @throws IOException if parsing fails */ public static Map parseJsonWithKeyCoercion(String json) throws IOException { - // First pass: everything is still strings for keys - Map intermediate = RELAXED_MAPPER.readValue(json, new TypeReference>() {}); - - // Second pass: coerce keys to proper types - Map result = new LinkedHashMap<>(); - for (Map.Entry entry : intermediate.entrySet()) { - result.put(coerceKey(entry.getKey()), entry.getValue()); - } - return result; - } - - /** - * Tries to turn a textual key into Integer, Long, Double, or Boolean. - * Falls back to the original String if none match. - * @param key The string key to coerce - * @return The coerced key or original string if no coercion is possible - */ - private static Object coerceKey(String key) { - try { - return Integer.valueOf(key); - } catch (NumberFormatException ignored) {} - - try { - return Long.valueOf(key); - } catch (NumberFormatException ignored) {} - - try { - return Double.valueOf(key); - } catch (NumberFormatException ignored) {} - - try { - return Float.valueOf(key); - } catch (NumberFormatException ignored) {} - - if ("true".equalsIgnoreCase(key) || "false".equalsIgnoreCase(key)) { - return Boolean.valueOf(key); - } - - return key; // leave it a String + return RELAXED_MAPPER.readValue(json, new TypeReference>() {}); } /** From 1f95323e30614e0904cab70f6c9ce6b59afd8d2a Mon Sep 17 00:00:00 2001 From: Rrutum <71127900+rrutum@users.noreply.github.com> Date: Tue, 15 Jul 2025 23:13:12 +0530 Subject: [PATCH 6/9] Map using relaxed mapper - wip --- .../java/com/aerospike/load/AsWriterTask.java | 47 ++---- .../com/aerospike/load/RelaxedJsonMapper.java | 136 +++++++++++++++++- .../aerospike/load/RelaxedJsonMapperTest.java | 41 +++++- 3 files changed, 184 insertions(+), 40 deletions(-) diff --git a/src/main/java/com/aerospike/load/AsWriterTask.java b/src/main/java/com/aerospike/load/AsWriterTask.java index f883875..c25cd4b 100644 --- a/src/main/java/com/aerospike/load/AsWriterTask.java +++ b/src/main/java/com/aerospike/load/AsWriterTask.java @@ -35,8 +35,7 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; import java.util.Map; - import com.fasterxml.jackson.databind.ObjectMapper; - import com.fasterxml.jackson.core.JsonProcessingException; + import com.aerospike.client.AerospikeClient; import com.aerospike.client.AerospikeException; @@ -446,38 +445,18 @@ private Bin createBinForJson(String binName, String binRawValue) { try { log.debug(binRawValue); - try { - ObjectMapper standardMapper = new ObjectMapper(); - Object obj = standardMapper.readValue(binRawValue, Object.class); - - if (obj instanceof List) { - List jsonArray = (List) obj; - return new Bin(binName, jsonArray); - } else if (obj instanceof Map) { - Map jsonObj = (Map) obj; - - if (this.params.unorderdMaps) { - return new Bin(binName, jsonObj); - } - - try { - TreeMap sortedMap = new TreeMap<>(); - sortedMap.putAll(jsonObj); - return new Bin(binName, sortedMap); - } catch (ClassCastException e) { - // Keys not comparable, fall back to unordered map - log.debug("TreeMap failed due to non-comparable keys, using unordered map: " + e.getMessage()); - return new Bin(binName, jsonObj); - } - } else { - return new Bin(binName, obj.toString()); - } - - } catch (JsonProcessingException standardParseException) { - // Standard JSON parsing failed, fall back to relaxed parser - log.debug("Standard JSON parsing failed, using relaxed parser: " + standardParseException.getMessage()); - Map relaxedResult = RelaxedJsonMapper.parseJsonWithKeyCoercion(binRawValue); - return new Bin(binName, relaxedResult); + // Parse with relaxed mapper, handling arrays and objects appropriately + Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(binRawValue, this.params.unorderdMaps); + + if (result instanceof List) { + // JSON array + return new Bin(binName, (List) result); + } else if (result instanceof Map) { + // JSON object (already ordered/unordered based on params) + return new Bin(binName, (Map) result); + } else { + // Primitive value + return new Bin(binName, result.toString()); } } catch (IOException e) { diff --git a/src/main/java/com/aerospike/load/RelaxedJsonMapper.java b/src/main/java/com/aerospike/load/RelaxedJsonMapper.java index 6e8f9d6..8fcd56f 100644 --- a/src/main/java/com/aerospike/load/RelaxedJsonMapper.java +++ b/src/main/java/com/aerospike/load/RelaxedJsonMapper.java @@ -78,7 +78,141 @@ public static List parseJsonToList(String json) throws IOException { * @throws IOException if parsing fails */ public static Map parseJsonWithKeyCoercion(String json) throws IOException { - return RELAXED_MAPPER.readValue(json, new TypeReference>() {}); + // First parse as a regular map with string keys + Map stringKeyMap = RELAXED_MAPPER.readValue(json, new TypeReference>() {}); + + // Create a new map with coerced keys + Map coercedMap = new LinkedHashMap<>(); + + for (Map.Entry entry : stringKeyMap.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + Object coercedKey = coerceKey(key); + coercedMap.put(coercedKey, value); + } + + return coercedMap; + } + + /** + * Coerce a string key to its appropriate type (Integer, Long, Double, Boolean, or String). + * @param key The string key to coerce + * @return The coerced key object + */ + private static Object coerceKey(String key) { + if (key == null) { + return key; + } + + // Try to parse as boolean + if ("true".equalsIgnoreCase(key)) { + return true; + } + if ("false".equalsIgnoreCase(key)) { + return false; + } + + // Try to parse as integer + try { + if (!key.contains(".") && !key.contains("e") && !key.contains("E")) { + long longValue = Long.parseLong(key); + if (longValue >= Integer.MIN_VALUE && longValue <= Integer.MAX_VALUE) { + return (int) longValue; + } else { + return longValue; + } + } + } catch (NumberFormatException e) { + // Not an integer + } + + // Try to parse as double + try { + return Double.parseDouble(key); + } catch (NumberFormatException e) { + // Not a double + } + + // Return as string + return key; + } + + /** + * Parse JSON string and return appropriate type based on content. + * For arrays: returns List + * For objects: returns Map (ordered or unordered based on parameter) + * @param json JSON string to parse + * @param unorderedMaps if true, return unordered maps; if false, return TreeMap for objects + * @return List for arrays, Map for objects, or primitive value + * @throws IOException if parsing fails + */ + public static Object parseJsonWithTypeHandling(String json, boolean unorderedMaps) throws IOException { + JsonNode node = RELAXED_MAPPER.readTree(json); + return convertJsonNodeWithKeyCoercion(node, unorderedMaps); + } + + /** + * Recursively convert a JsonNode to Java objects with key coercion applied. + * @param node The JsonNode to convert + * @param unorderedMaps if true, return unordered maps; if false, return TreeMap for objects + * @return The converted Java object with coerced keys + */ + private static Object convertJsonNodeWithKeyCoercion(JsonNode node, boolean unorderedMaps) { + if (node == null || node.isNull()) { + return null; + } else if (node.isBoolean()) { + return node.asBoolean(); + } else if (node.isInt()) { + return node.asInt(); + } else if (node.isLong()) { + return node.asLong(); + } else if (node.isFloat()) { + return node.floatValue(); + } else if (node.isDouble()) { + return node.asDouble(); + } else if (node.isTextual()) { + return node.asText(); + } else if (node.isArray()) { + // Process array elements recursively + java.util.List list = new java.util.ArrayList<>(); + for (JsonNode element : node) { + list.add(convertJsonNodeWithKeyCoercion(element, unorderedMaps)); + } + return list; + } else if (node.isObject()) { + // Process object with key coercion and recursive value processing + Map map = unorderedMaps ? + new LinkedHashMap<>() : new LinkedHashMap<>(); // Start with LinkedHashMap, convert to TreeMap later if needed + + node.fields().forEachRemaining(entry -> { + String stringKey = entry.getKey(); + JsonNode valueNode = entry.getValue(); + + // Coerce the key + Object coercedKey = coerceKey(stringKey); + + // Recursively process the value + Object processedValue = convertJsonNodeWithKeyCoercion(valueNode, unorderedMaps); + + map.put(coercedKey, processedValue); + }); + + // If ordered maps are requested, try to create a TreeMap + if (!unorderedMaps) { + try { + java.util.TreeMap sortedMap = new java.util.TreeMap<>(); + sortedMap.putAll(map); + return sortedMap; + } catch (ClassCastException e) { + // Keys not comparable, return unordered map + return map; + } + } + + return map; + } + + return node.toString(); } /** diff --git a/src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java b/src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java index 5294953..dba7aaa 100644 --- a/src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java +++ b/src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java @@ -33,14 +33,45 @@ public void testParseJsonWithKeyCoercion() throws Exception { @Test public void testParseJsonToList() throws Exception { - String json = "[1, 2, 'three', true]"; - List list = RelaxedJsonMapper.parseJsonToList(json); + String json = "[1, 'two', true, {key: 'value'}]"; + List result = RelaxedJsonMapper.parseJsonToList(json); - assertEquals(4, list.size()); + assertEquals(4, result.size()); + assertEquals(1, result.get(0)); + assertEquals("two", result.get(1)); + assertEquals(true, result.get(2)); + + @SuppressWarnings("unchecked") + Map obj = (Map) result.get(3); + assertEquals("value", obj.get("key")); + } + + @Test + public void testNestedObjectsInArraysWithNumericKeys() throws Exception { + String json = "[1, 2, {1: 'a', 2: 'b', true: 'flag'}]"; + Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(json, true); + + assertTrue(result instanceof List); + @SuppressWarnings("unchecked") + List list = (List) result; + + assertEquals(3, list.size()); assertEquals(1, list.get(0)); assertEquals(2, list.get(1)); - assertEquals("three", list.get(2)); - assertEquals(true, list.get(3)); + + // Check the nested object + assertTrue(list.get(2) instanceof Map); + @SuppressWarnings("unchecked") + Map nestedMap = (Map) list.get(2); + + // Verify that numeric keys are preserved as numbers, not strings + assertEquals("a", nestedMap.get(1)); // Key should be integer 1, not string "1" + assertEquals("b", nestedMap.get(2)); // Key should be integer 2, not string "2" + assertEquals("flag", nestedMap.get(true)); // Key should be boolean true + + // Verify that string keys would not work (they should be coerced to numbers) + assertNull(nestedMap.get("1")); // String "1" should not exist + assertNull(nestedMap.get("2")); // String "2" should not exist } @Test From c801ec470abee973709833e64df929e5cec136ea Mon Sep 17 00:00:00 2001 From: Rrutum <71127900+rrutum@users.noreply.github.com> Date: Wed, 16 Jul 2025 11:00:15 +0530 Subject: [PATCH 7/9] Move ordering logic out of mapper --- .../java/com/aerospike/load/AsWriterTask.java | 31 +++- .../com/aerospike/load/RelaxedJsonMapper.java | 145 ++++++++++-------- .../aerospike/load/RelaxedJsonMapperTest.java | 36 ++++- 3 files changed, 142 insertions(+), 70 deletions(-) diff --git a/src/main/java/com/aerospike/load/AsWriterTask.java b/src/main/java/com/aerospike/load/AsWriterTask.java index c25cd4b..c14a778 100644 --- a/src/main/java/com/aerospike/load/AsWriterTask.java +++ b/src/main/java/com/aerospike/load/AsWriterTask.java @@ -444,16 +444,33 @@ private Bin createBinForString(String binName, String binRawValue) { private Bin createBinForJson(String binName, String binRawValue) { try { log.debug(binRawValue); - - // Parse with relaxed mapper, handling arrays and objects appropriately - Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(binRawValue, this.params.unorderdMaps); - + log.info("Parsing JSON for bin: " + binName + " with raw value: " + binRawValue); + // Parse with relaxed mapper (always returns unordered maps) + Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(binRawValue); + if (result instanceof List) { - // JSON array + // JSON array - no ordering needed return new Bin(binName, (List) result); } else if (result instanceof Map) { - // JSON object (already ordered/unordered based on params) - return new Bin(binName, (Map) result); + // JSON object - apply ordering logic based on params + Map jsonMap = (Map) result; + + if (this.params.unorderdMaps) { + // Return unordered map as-is + return new Bin(binName, jsonMap); + } else { + // Try to create ordered TreeMap + try { + @SuppressWarnings("unchecked") + Map castedMap = (Map) jsonMap; + TreeMap sortedMap = new TreeMap<>(castedMap); + return new Bin(binName, sortedMap); + } catch (ClassCastException e) { + // Keys not comparable, fall back to unordered map + log.warn("TreeMap failed due to non-comparable keys, using unordered map: {}", e.getMessage()); + return new Bin(binName, jsonMap); + } + } } else { // Primitive value return new Bin(binName, result.toString()); diff --git a/src/main/java/com/aerospike/load/RelaxedJsonMapper.java b/src/main/java/com/aerospike/load/RelaxedJsonMapper.java index 8fcd56f..bab5257 100644 --- a/src/main/java/com/aerospike/load/RelaxedJsonMapper.java +++ b/src/main/java/com/aerospike/load/RelaxedJsonMapper.java @@ -5,6 +5,8 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.io.FileReader; import java.io.IOException; @@ -20,6 +22,7 @@ public class RelaxedJsonMapper { private static final ObjectMapper RELAXED_MAPPER = new ObjectMapper(); private static final ObjectMapper STANDARD_MAPPER = new ObjectMapper(); + private static final Logger log = LogManager.getLogger(RelaxedJsonMapper.class); static { // Configure relaxed mapper to allow JSON supersets @@ -140,81 +143,99 @@ private static Object coerceKey(String key) { /** * Parse JSON string and return appropriate type based on content. * For arrays: returns List - * For objects: returns Map (ordered or unordered based on parameter) + * For objects: returns Map (always unordered LinkedHashMap) * @param json JSON string to parse - * @param unorderedMaps if true, return unordered maps; if false, return TreeMap for objects * @return List for arrays, Map for objects, or primitive value * @throws IOException if parsing fails */ - public static Object parseJsonWithTypeHandling(String json, boolean unorderedMaps) throws IOException { - JsonNode node = RELAXED_MAPPER.readTree(json); - return convertJsonNodeWithKeyCoercion(node, unorderedMaps); + public static Object parseJsonWithTypeHandling(String json) throws IOException { + JsonNode root = RELAXED_MAPPER.readTree(json); + + if (root.isArray()) { + List list = RELAXED_MAPPER.convertValue(root, new TypeReference>() {}); + // Walk list and coerce any map keys in nested objects + return coerceKeysInList(list); +// return list; + } else if (root.isObject()) { + Map map = RELAXED_MAPPER.convertValue(root, new TypeReference>() {}); + // Coerce keys to Integer/Long/Boolean/Double and build Map +// return coerceKeysInMap(map); + for (Map.Entry entry : map.entrySet()) { + log.info("Map entry: " + entry.getKey() + " -> " + entry.getValue()); + log.info("Map types: " + entry.getKey().getClass().getSimpleName() + " -> " + entry.getValue().getClass().getSimpleName()); + } + return map; + } else { + // Primitive value + return RELAXED_MAPPER.convertValue(root, Object.class); + } } /** - * Recursively convert a JsonNode to Java objects with key coercion applied. - * @param node The JsonNode to convert - * @param unorderedMaps if true, return unordered maps; if false, return TreeMap for objects - * @return The converted Java object with coerced keys + * Recursively coerce keys in a list (for nested objects within arrays). + * @param list The list to process + * @return List with coerced keys in any nested objects */ - private static Object convertJsonNodeWithKeyCoercion(JsonNode node, boolean unorderedMaps) { - if (node == null || node.isNull()) { - return null; - } else if (node.isBoolean()) { - return node.asBoolean(); - } else if (node.isInt()) { - return node.asInt(); - } else if (node.isLong()) { - return node.asLong(); - } else if (node.isFloat()) { - return node.floatValue(); - } else if (node.isDouble()) { - return node.asDouble(); - } else if (node.isTextual()) { - return node.asText(); - } else if (node.isArray()) { - // Process array elements recursively - java.util.List list = new java.util.ArrayList<>(); - for (JsonNode element : node) { - list.add(convertJsonNodeWithKeyCoercion(element, unorderedMaps)); - } - return list; - } else if (node.isObject()) { - // Process object with key coercion and recursive value processing - Map map = unorderedMaps ? - new LinkedHashMap<>() : new LinkedHashMap<>(); // Start with LinkedHashMap, convert to TreeMap later if needed - - node.fields().forEachRemaining(entry -> { - String stringKey = entry.getKey(); - JsonNode valueNode = entry.getValue(); - - // Coerce the key - Object coercedKey = coerceKey(stringKey); - - // Recursively process the value - Object processedValue = convertJsonNodeWithKeyCoercion(valueNode, unorderedMaps); - - map.put(coercedKey, processedValue); - }); - - // If ordered maps are requested, try to create a TreeMap - if (!unorderedMaps) { - try { - java.util.TreeMap sortedMap = new java.util.TreeMap<>(); - sortedMap.putAll(map); - return sortedMap; - } catch (ClassCastException e) { - // Keys not comparable, return unordered map - return map; + private static List coerceKeysInList(List list) { + List result = new java.util.ArrayList<>(); + for (Object item : list) { + log.info("Processing item: " + item); + log.info("Item type: " + (item != null ? item.getClass().getSimpleName() : "null")); + if (item instanceof Map) { + for (Map.Entry entry : ((Map) item).entrySet()) { + log.info("Map entry: " + entry.getKey() + " -> " + entry.getValue()); + log.info("Map entry: " + entry.getKey().getClass().getSimpleName() + " -> " + entry.getValue().getClass().getSimpleName()); } +// @SuppressWarnings("unchecked") +// Map mapItem = (Map) item; +// result.add(coerceKeysInMap(mapItem)); + result.add(item); + } else if (item instanceof List) { + @SuppressWarnings("unchecked") + List listItem = (List) item; + result.add(coerceKeysInList(listItem)); + } else { + result.add(item); } - - return map; } - - return node.toString(); + return result; } + /** + * Coerce string keys to appropriate types and recursively process nested structures. + * @param map The map to process + * @return Map with coerced keys + */ +// private static Map coerceKeysInMap(Map map) { +// Map result = new LinkedHashMap<>(); // Preserves insertion order +// +// for (Map.Entry entry : map.entrySet()) { +// Object stringKey = entry.getKey(); +// Object value = entry.getValue(); +// +// // Coerce the key +// Object coercedKey = coerceKey(stringKey); +// +// // Recursively process nested structures +// Object processedValue; +// if (value instanceof Map) { +// @SuppressWarnings("unchecked") +// Map mapValue = (Map) value; +// processedValue = coerceKeysInMap(mapValue); +// } else if (value instanceof List) { +// @SuppressWarnings("unchecked") +// List listValue = (List) value; +// processedValue = coerceKeysInList(listValue); +// } else { +// processedValue = value; +// } +// +// result.put(coercedKey, processedValue); +// } +// +// return result; +// } + /** * Convert a JsonNode to a Java object (Map, List, or primitive). * @param node The JsonNode to convert diff --git a/src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java b/src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java index dba7aaa..f689dc5 100644 --- a/src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java +++ b/src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java @@ -49,7 +49,7 @@ public void testParseJsonToList() throws Exception { @Test public void testNestedObjectsInArraysWithNumericKeys() throws Exception { String json = "[1, 2, {1: 'a', 2: 'b', true: 'flag'}]"; - Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(json, true); + Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(json); assertTrue(result instanceof List); @SuppressWarnings("unchecked") @@ -74,6 +74,40 @@ public void testNestedObjectsInArraysWithNumericKeys() throws Exception { assertNull(nestedMap.get("2")); // String "2" should not exist } + @Test + public void testFieldOrderPreservation() throws Exception { + // Test the specific case mentioned by user: {2:"a","1":"b"} + String json = "{2: 'a', 1: 'b', 3: 'c'}"; + Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(json); + + assertTrue(result instanceof Map); + @SuppressWarnings("unchecked") + Map map = (Map) result; + + // Verify values are correct + assertEquals("a", map.get(2)); + assertEquals("b", map.get(1)); + assertEquals("c", map.get(3)); + + // Verify that the map preserves insertion order + // Convert to array to check order + Object[] keys = map.keySet().toArray(); + Object[] values = map.values().toArray(); + + // Should be in insertion order: 2, 1, 3 + assertEquals(2, keys[0]); + assertEquals(1, keys[1]); + assertEquals(3, keys[2]); + + assertEquals("a", values[0]); + assertEquals("b", values[1]); + assertEquals("c", values[2]); + + System.out.println("Original JSON: " + json); + System.out.println("Parsed result keys in order: " + java.util.Arrays.toString(keys)); + System.out.println("Parsed result values in order: " + java.util.Arrays.toString(values)); + } + @Test public void testParseJsonToMap() throws Exception { String json = "{key1: 'value1', key2: 42, key3: true}"; From 3aa2394e81e67b0e430f38bd2fd85e80ac170a5c Mon Sep 17 00:00:00 2001 From: Rrutum <71127900+rrutum@users.noreply.github.com> Date: Wed, 16 Jul 2025 11:09:36 +0530 Subject: [PATCH 8/9] Coerce keys in map --- .../com/aerospike/load/RelaxedJsonMapper.java | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/src/main/java/com/aerospike/load/RelaxedJsonMapper.java b/src/main/java/com/aerospike/load/RelaxedJsonMapper.java index bab5257..f165e0b 100644 --- a/src/main/java/com/aerospike/load/RelaxedJsonMapper.java +++ b/src/main/java/com/aerospike/load/RelaxedJsonMapper.java @@ -157,14 +157,14 @@ public static Object parseJsonWithTypeHandling(String json) throws IOException { return coerceKeysInList(list); // return list; } else if (root.isObject()) { - Map map = RELAXED_MAPPER.convertValue(root, new TypeReference>() {}); + Map map = RELAXED_MAPPER.convertValue(root, new TypeReference>() {}); // Coerce keys to Integer/Long/Boolean/Double and build Map -// return coerceKeysInMap(map); - for (Map.Entry entry : map.entrySet()) { - log.info("Map entry: " + entry.getKey() + " -> " + entry.getValue()); - log.info("Map types: " + entry.getKey().getClass().getSimpleName() + " -> " + entry.getValue().getClass().getSimpleName()); - } - return map; + return coerceKeysInMap(map); +// for (Map.Entry entry : map.entrySet()) { +// log.info("Map entry: " + entry.getKey() + " -> " + entry.getValue()); +// log.info("Map types: " + entry.getKey().getClass().getSimpleName() + " -> " + entry.getValue().getClass().getSimpleName()); +// } +// return map; } else { // Primitive value return RELAXED_MAPPER.convertValue(root, Object.class); @@ -206,35 +206,35 @@ private static List coerceKeysInList(List list) { * @param map The map to process * @return Map with coerced keys */ -// private static Map coerceKeysInMap(Map map) { -// Map result = new LinkedHashMap<>(); // Preserves insertion order -// -// for (Map.Entry entry : map.entrySet()) { -// Object stringKey = entry.getKey(); -// Object value = entry.getValue(); -// -// // Coerce the key -// Object coercedKey = coerceKey(stringKey); -// -// // Recursively process nested structures -// Object processedValue; -// if (value instanceof Map) { -// @SuppressWarnings("unchecked") -// Map mapValue = (Map) value; -// processedValue = coerceKeysInMap(mapValue); -// } else if (value instanceof List) { -// @SuppressWarnings("unchecked") -// List listValue = (List) value; -// processedValue = coerceKeysInList(listValue); -// } else { -// processedValue = value; -// } -// -// result.put(coercedKey, processedValue); -// } -// -// return result; -// } + private static Map coerceKeysInMap(Map map) { + Map result = new LinkedHashMap<>(); // Preserves insertion order + + for (Map.Entry entry : map.entrySet()) { + String stringKey = entry.getKey(); + Object value = entry.getValue(); + + // Coerce the key + Object coercedKey = coerceKey(stringKey); + + // Recursively process nested structures + Object processedValue; + if (value instanceof Map) { + @SuppressWarnings("unchecked") + Map mapValue = (Map) value; + processedValue = coerceKeysInMap(mapValue); + } else if (value instanceof List) { + @SuppressWarnings("unchecked") + List listValue = (List) value; + processedValue = coerceKeysInList(listValue); + } else { + processedValue = value; + } + + result.put(coercedKey, processedValue); + } + + return result; + } /** * Convert a JsonNode to a Java object (Map, List, or primitive). From d56cd1aa7d2f21a3a1508c9ea30cabbf61165769 Mon Sep 17 00:00:00 2001 From: Rrutum <71127900+rrutum@users.noreply.github.com> Date: Thu, 17 Jul 2025 16:38:44 +0530 Subject: [PATCH 9/9] Parse invalid json with best effort --- .../java/com/aerospike/load/AsWriterTask.java | 58 ++--- .../com/aerospike/load/RelaxedJsonMapper.java | 83 +------ .../java/com/aerospike/load/DataTypeTest.java | 122 +++++------ .../aerospike/load/RelaxedJsonMapperTest.java | 202 +++++++++++------- 4 files changed, 221 insertions(+), 244 deletions(-) diff --git a/src/main/java/com/aerospike/load/AsWriterTask.java b/src/main/java/com/aerospike/load/AsWriterTask.java index c14a778..a34d8de 100644 --- a/src/main/java/com/aerospike/load/AsWriterTask.java +++ b/src/main/java/com/aerospike/load/AsWriterTask.java @@ -35,7 +35,8 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; import java.util.Map; - + import com.fasterxml.jackson.databind.ObjectMapper; + import com.fasterxml.jackson.core.JsonProcessingException; import com.aerospike.client.AerospikeClient; import com.aerospike.client.AerospikeException; @@ -444,36 +445,39 @@ private Bin createBinForString(String binName, String binRawValue) { private Bin createBinForJson(String binName, String binRawValue) { try { log.debug(binRawValue); - log.info("Parsing JSON for bin: " + binName + " with raw value: " + binRawValue); - // Parse with relaxed mapper (always returns unordered maps) - Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(binRawValue); + + Object parsedObject = null; + + try { + ObjectMapper standardMapper = new ObjectMapper(); + parsedObject = standardMapper.readValue(binRawValue, Object.class); + log.debug("Successfully parsed with standard JSON parser"); + } catch (JsonProcessingException standardParseException) { + log.debug("Standard JSON parsing failed, using relaxed parser: " + standardParseException.getMessage()); + parsedObject = RelaxedJsonMapper.parseJsonWithTypeHandling(binRawValue); + } + + // Apply the same logic regardless of which parser was used + if (parsedObject instanceof List) { + List jsonArray = (List) parsedObject; + return new Bin(binName, jsonArray); + } else if (parsedObject instanceof Map) { + Map jsonObj = (Map) parsedObject; - if (result instanceof List) { - // JSON array - no ordering needed - return new Bin(binName, (List) result); - } else if (result instanceof Map) { - // JSON object - apply ordering logic based on params - Map jsonMap = (Map) result; - if (this.params.unorderdMaps) { - // Return unordered map as-is - return new Bin(binName, jsonMap); - } else { - // Try to create ordered TreeMap - try { - @SuppressWarnings("unchecked") - Map castedMap = (Map) jsonMap; - TreeMap sortedMap = new TreeMap<>(castedMap); - return new Bin(binName, sortedMap); - } catch (ClassCastException e) { - // Keys not comparable, fall back to unordered map - log.warn("TreeMap failed due to non-comparable keys, using unordered map: {}", e.getMessage()); - return new Bin(binName, jsonMap); - } + return new Bin(binName, jsonObj); + } + + try { + TreeMap sortedMap = new TreeMap<>(); + sortedMap.putAll(jsonObj); + return new Bin(binName, sortedMap); + } catch (ClassCastException e) { + log.debug("TreeMap failed due to non-comparable keys, using unordered map: " + e.getMessage()); + return new Bin(binName, jsonObj); } } else { - // Primitive value - return new Bin(binName, result.toString()); + return new Bin(binName, parsedObject.toString()); } } catch (IOException e) { diff --git a/src/main/java/com/aerospike/load/RelaxedJsonMapper.java b/src/main/java/com/aerospike/load/RelaxedJsonMapper.java index f165e0b..97769cf 100644 --- a/src/main/java/com/aerospike/load/RelaxedJsonMapper.java +++ b/src/main/java/com/aerospike/load/RelaxedJsonMapper.java @@ -43,60 +43,6 @@ public static JsonNode parseJson(FileReader reader) throws IOException { return RELAXED_MAPPER.readTree(reader); } - /** - * Parse JSON from a string. - * @param json JSON string - * @return JsonNode representing the parsed JSON - * @throws IOException if parsing fails - */ - public static JsonNode parseJson(String json) throws IOException { - return RELAXED_MAPPER.readTree(json); - } - - /** - * Parse JSON string into a Map. - * @param json JSON string - * @return Map representation of the JSON - * @throws IOException if parsing fails - */ - public static Map parseJsonToMap(String json) throws IOException { - return RELAXED_MAPPER.readValue(json, new TypeReference>() {}); - } - - /** - * Parse JSON string into a List. - * @param json JSON string - * @return List representation of the JSON - * @throws IOException if parsing fails - */ - public static List parseJsonToList(String json) throws IOException { - return RELAXED_MAPPER.readValue(json, new TypeReference>() {}); - } - - /** - * Parse JSON with automatic key type coercion (similar to the sample code). - * This method converts string keys to appropriate types (Integer, Long, Double, Boolean). - * @param json JSON string - * @return Map with coerced keys - * @throws IOException if parsing fails - */ - public static Map parseJsonWithKeyCoercion(String json) throws IOException { - // First parse as a regular map with string keys - Map stringKeyMap = RELAXED_MAPPER.readValue(json, new TypeReference>() {}); - - // Create a new map with coerced keys - Map coercedMap = new LinkedHashMap<>(); - - for (Map.Entry entry : stringKeyMap.entrySet()) { - String key = entry.getKey(); - Object value = entry.getValue(); - Object coercedKey = coerceKey(key); - coercedMap.put(coercedKey, value); - } - - return coercedMap; - } - /** * Coerce a string key to its appropriate type (Integer, Long, Double, Boolean, or String). * @param key The string key to coerce @@ -155,16 +101,10 @@ public static Object parseJsonWithTypeHandling(String json) throws IOException { List list = RELAXED_MAPPER.convertValue(root, new TypeReference>() {}); // Walk list and coerce any map keys in nested objects return coerceKeysInList(list); -// return list; } else if (root.isObject()) { Map map = RELAXED_MAPPER.convertValue(root, new TypeReference>() {}); // Coerce keys to Integer/Long/Boolean/Double and build Map return coerceKeysInMap(map); -// for (Map.Entry entry : map.entrySet()) { -// log.info("Map entry: " + entry.getKey() + " -> " + entry.getValue()); -// log.info("Map types: " + entry.getKey().getClass().getSimpleName() + " -> " + entry.getValue().getClass().getSimpleName()); -// } -// return map; } else { // Primitive value return RELAXED_MAPPER.convertValue(root, Object.class); @@ -179,17 +119,10 @@ public static Object parseJsonWithTypeHandling(String json) throws IOException { private static List coerceKeysInList(List list) { List result = new java.util.ArrayList<>(); for (Object item : list) { - log.info("Processing item: " + item); - log.info("Item type: " + (item != null ? item.getClass().getSimpleName() : "null")); if (item instanceof Map) { - for (Map.Entry entry : ((Map) item).entrySet()) { - log.info("Map entry: " + entry.getKey() + " -> " + entry.getValue()); - log.info("Map entry: " + entry.getKey().getClass().getSimpleName() + " -> " + entry.getValue().getClass().getSimpleName()); - } -// @SuppressWarnings("unchecked") -// Map mapItem = (Map) item; -// result.add(coerceKeysInMap(mapItem)); - result.add(item); + @SuppressWarnings("unchecked") + Map mapItem = (Map) item; + result.add(coerceKeysInMap(mapItem)); } else if (item instanceof List) { @SuppressWarnings("unchecked") List listItem = (List) item; @@ -277,16 +210,6 @@ public static Object getFromJsonNode(JsonNode node, String fieldName) { return jsonNodeToObject(node.get(fieldName)); } - /** - * Check if a JsonNode has a specific field. - * @param node The JsonNode to check - * @param fieldName The name of the field - * @return true if the field exists, false otherwise - */ - public static boolean hasField(JsonNode node, String fieldName) { - return node != null && node.has(fieldName); - } - /** * Get a JsonNode as a string representation. * @param node The JsonNode to convert diff --git a/src/test/java/com/aerospike/load/DataTypeTest.java b/src/test/java/com/aerospike/load/DataTypeTest.java index 86a8506..d6aaefb 100644 --- a/src/test/java/com/aerospike/load/DataTypeTest.java +++ b/src/test/java/com/aerospike/load/DataTypeTest.java @@ -47,9 +47,9 @@ enum BinType { * */ public class DataTypeTest { - + String host = "127.0.0.1"; - String port = "3100"; + String port = "3000"; String ns = "test"; String set = null; MapOrder expectedMapOrder = MapOrder.KEY_ORDERED; @@ -83,7 +83,7 @@ public void setUp() { public void tearDown() { client.close(); } - + public List> parseDataFile(String dataFile) { BufferedReader br = null; String delimiter = ","; @@ -103,7 +103,7 @@ public List> parseDataFile(String dataFile) { } return recordDataList; } - + public JsonNode parseConfigFile(String configFile) { JsonNode jsonNode = null; try{ @@ -127,7 +127,7 @@ private HashMap getMapFromJsonNode(JsonNode node, String key) { } return new HashMap<>(); } - + // String type data validation //@Test public void testValidateString() throws Exception { @@ -143,19 +143,19 @@ public void testValidateString() throws Exception { int setMod = 5, range = 100, seed = 10, nrecords = 10; dataFile = rootDir + "dataString.dsv"; - writeDataMap(dataFile, nrecords, setMod, range, seed, binMap); - - // Run Aerospike loader + writeDataMap(dataFile, nrecords, setMod, range, seed, binMap); + + // Run Aerospike loader AerospikeLoad.main(new String[]{"-h", host,"-p", port,"-n", ns, "-ec", error_count,"-wa", write_action,"-c", "src/test/resources/configString.json", dataFile}); - + // Validate loaded data String dstType = null; boolean dataValid = validateMap(client, dataFile, nrecords, setMod, range, seed, binMap, dstType); boolean error = getError(log); - + assertTrue(dataValid); assertTrue(!error); - + System.out.println("TestValidateString: Complete"); } @@ -167,7 +167,7 @@ public void testValidateInteger() throws Exception { System.out.println("Client is not able to connect:" + host + ":" + port); return; } - + // Create datafile HashMap binMap = getMapFromJsonNode(testSchema, "test_integer"); @@ -175,19 +175,19 @@ public void testValidateInteger() throws Exception { int setMod = 5, range = 100, seed = 10, nrecords = 10; dataFile = rootDir + "dataInt.dsv"; - writeDataMap(dataFile, nrecords, setMod, range, seed, binMap); - + writeDataMap(dataFile, nrecords, setMod, range, seed, binMap); + // Run Aerospike loader AerospikeLoad.main(new String[]{"-h", host,"-p", port,"-n", ns, "-ec", error_count,"-wa", write_action,"-c", "src/test/resources/configInt.json", dataFile}); - + // Validate loaded data String dstType = null; boolean dataValid = validateMap(client, dataFile, nrecords, setMod, range, seed, binMap, dstType); boolean error = getError(log); - + assertTrue(dataValid); assertTrue(!error); - + System.out.println("TestValidateInteger: Complete"); } @@ -199,7 +199,7 @@ public void testValidateStringUtf8() throws Exception { System.out.println("Client is not able to connect:" + host + ":" + port); return; } - + // Create datafile HashMap binMap = getMapFromJsonNode(testSchema, "test_utf8"); @@ -208,18 +208,18 @@ public void testValidateStringUtf8() throws Exception { int setMod = 5, range = 100, seed = 10, nrecords = 10; dataFile = rootDir + "dataUtf8.dsv"; writeDataMap(dataFile, nrecords, setMod, range, seed, binMap); - + // Run Aerospike loader AerospikeLoad.main(new String[]{"-h", host,"-p", port,"-n", ns, "-ec", error_count,"-wa", write_action,"-c", "src/test/resources/configUtf8.json", dataFile}); - + // Validate loaded data String dstType = null; boolean dataValid = validateMap(client, dataFile, nrecords, setMod, range, seed, binMap, dstType); boolean error = getError(log); - + assertTrue(dataValid); assertTrue(!error); - + System.out.println("TestValidateStringutf8: Complete"); } @@ -235,22 +235,22 @@ public void testValidateTimestampInteger() throws Exception { HashMap binMap = getMapFromJsonNode(testSchema, "test_date"); - + int setMod = 5, range = 100, seed = 10, nrecords = 10; dataFile = rootDir + "dataDate.dsv"; writeDataMap(dataFile, nrecords, setMod, range, seed, binMap); - + // Run Aerospike loader AerospikeLoad.main(new String[]{"-h", host,"-p", port,"-n", ns, "-ec", error_count,"-wa", write_action,"-c", "src/test/resources/configDate.json", dataFile}); - + // Validate loaded data - String dst_type = "integer"; + String dst_type = "integer"; boolean dataValid = validateMap(client, dataFile, nrecords, setMod, range, seed, binMap, dst_type); boolean error = getError(log); - + assertTrue(dataValid); assertTrue(!error); - + System.out.println("TestValidateTimestampInteger: Complete"); } @@ -266,22 +266,22 @@ public void testValidateBlob() throws Exception { HashMap binMap = getMapFromJsonNode(testSchema, "test_blob"); - + int setMod = 5, range = 100, seed = 10, nrecords = 10; dataFile = rootDir + "dataBlob.dsv"; writeDataMap(dataFile, nrecords, setMod, range, seed, binMap); - - // Run Aerospike loader + + // Run Aerospike loader AerospikeLoad.main(new String[]{"-h", host,"-p", port,"-n", ns, "-ec", error_count,"-wa", write_action,"-c", "src/test/resources/configBlob.json", dataFile}); - + // Validate loaded data String dstType = "blob"; boolean dataValid = validateMap(client, dataFile, nrecords, setMod, range, seed, binMap, dstType); boolean error = getError(log); - + assertTrue(dataValid); assertTrue(!error); - + System.out.println("TestValidateBlob: Complete"); } @@ -297,14 +297,14 @@ public void testValidateList() throws Exception { HashMap binMap = getMapFromJsonNode(testSchema, "test_list"); - + int setMod = 5, range = 100, seed = 10, nrecords = 10; dataFile = rootDir + "dataList.dsv"; writeDataMap(dataFile, nrecords, setMod, range, seed, binMap); - + // Run Aerospike loader AerospikeLoad.main(new String[]{"-h", host,"-p", port,"-n", ns, "-ec", error_count,"-wa", write_action,"-c", "src/test/resources/configList.json", dataFile}); - + // Validate loaded data String dstType = "list"; boolean dataValid = validateMap(client, dataFile, nrecords, setMod, range, seed, binMap, dstType); @@ -324,19 +324,19 @@ public void testValidateMap() throws Exception { System.out.println("Client is not able to connect:" + host + ":" + port); return; } - + // Create datafile HashMap binMap = getMapFromJsonNode(testSchema, "test_map"); - + int setMod = 5, range = 100, seed = 10, nrecords = 10; dataFile = rootDir + "dataMap.dsv"; writeDataMap(dataFile, nrecords, setMod, range, seed, binMap); - + // Run Aerospike loader AerospikeLoad.main(new String[]{"-h", host,"-p", port,"-n", ns, "-ec", error_count,"-wa", write_action,"-c", "src/test/resources/configMap.json", dataFile}); - + // Validate loaded data String dstType = "map"; boolean dataValid = validateMap(client, dataFile, nrecords, setMod, range, seed, binMap, dstType); @@ -361,14 +361,14 @@ public void testValidateJSON() throws Exception { HashMap binMap = getMapFromJsonNode(testSchema, "test_json"); - + int setMod = 5, range = 100, seed = 10, nrecords = 10; dataFile = rootDir + "dataJson.dsv"; writeDataMap(dataFile, nrecords, setMod, range, seed, binMap); - + // Run Aerospike loader AerospikeLoad.main(new String[]{"-h", host,"-p", port,"-n", ns, "-v", "-ec", error_count,"-wa", write_action,"-c", "src/test/resources/configJson.json", dataFile}); - + // Validate loaded data String dstType = "json"; boolean dataValid = validateMap(client, dataFile, nrecords, setMod, range, seed, binMap, dstType); @@ -393,18 +393,18 @@ public void testAllDatatype() throws Exception { HashMap binMap = getMapFromJsonNode(testSchema, "test_alltype"); - + int setMod = 5, range = 100, seed = 10, nrecords = 10; dataFile = rootDir + "configAllDataType.dsv"; writeDataMap(dataFile, nrecords, setMod, range, seed, binMap); - + // Run Aerospike loader AerospikeLoad.main(new String[]{"-h", host,"-p", port,"-n", ns, "-ec", error_count,"-wa", write_action,"-c", "src/test/resources/configAllDataType.json", dataFile}); - + boolean error = getError(log); assertTrue(!error); - + System.out.println("TestAllDatatype: Complete"); } @@ -416,13 +416,13 @@ public void testDynamicBinName() throws Exception { System.out.println("Client is not able to connect:" + host + ":" + port); return; } - + AerospikeLoad.main(new String[]{"-h", host,"-p", port,"-n", ns, "-ec", error_count,"-wa", write_action,"-c", "src/test/resources/configDynamicBinName.json", "src/test/resources/dataDynamicBin.csv"}); boolean error = getError(log); assertTrue(!error); - + System.out.println("Test Dynamic BinName: Complete"); } @@ -434,13 +434,13 @@ public void testStaticBinName() throws Exception { System.out.println("Client is not able to connect:" + host + ":" + port); return; } - + AerospikeLoad.main(new String[]{"-h", host,"-p", port,"-n", ns, "-ec", error_count,"-wa", write_action,"-c", "src/test/resources/configStaticBinName.json", "src/test/resources/dataStaticBin.csv"}); boolean error = getError(log); assertTrue(!error); - + System.out.println("Test static BinName: Complete"); } @@ -452,20 +452,20 @@ public void testValidateMapOrder() throws Exception { System.out.println("Client is not able to connect:" + host + ":" + port); return; } - + // Create datafile HashMap binMap = getMapFromJsonNode(testSchema, "test_map"); - + int setMod = 5, range = 100, seed = 10, nrecords = 10; dataFile = rootDir + "dataMap.dsv"; writeDataMap(dataFile, nrecords, setMod, range, seed, binMap); - + // Run Aerospike loader this.expectedMapOrder = MapOrder.UNORDERED; AerospikeLoad.main(new String[]{"-h", host,"-p", port,"-n", ns, "-ec", error_count,"-wa", write_action, "-um", "-c", "src/test/resources/configMap.json", dataFile}); - + // Validate loaded data String dstType = "map"; boolean dataValid = validateMap(client, dataFile, nrecords, setMod, range, seed, binMap, dstType); @@ -549,7 +549,7 @@ public boolean validateMap(AerospikeClient client, String filename, int nrecords Bin bin1 = null; String bin1Type = null; Record record = null; - + rint = r.nextInt(range); Iterator> iterator = binMap.entrySet().iterator(); @@ -630,7 +630,7 @@ private boolean validateBin(Key key, Bin bin, String binType, String dstType, Re } catch (java.text.ParseException e) { e.printStackTrace(); } - + } else if (dstType != null && dstType.equalsIgnoreCase("blob")) { expected = convertHexToString(bin.value.toString()); received = new String((byte[]) received); @@ -664,7 +664,7 @@ private boolean validateBin(Key key, Bin bin, String binType, String dstType, Re } else { expected = bin.value.toString(); } - + if (received != null && received.toString().equals(expected)) { System.out.println(String.format( "Bin matched: namespace=%s set=%s key=%s bin=%s value=%s generation=%d expiration=%d", @@ -678,7 +678,7 @@ private boolean validateBin(Key key, Bin bin, String binType, String dstType, Re return valid; } - + /** * @param log log file name * @return return true if get any error @@ -710,7 +710,7 @@ public boolean getError(String log) { } return error; } - + /** * @param binName binName prefix * @param binType type of binValue diff --git a/src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java b/src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java index f689dc5..48b4876 100644 --- a/src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java +++ b/src/test/java/com/aerospike/load/RelaxedJsonMapperTest.java @@ -3,150 +3,200 @@ import org.junit.Test; import static org.junit.Assert.*; -import com.fasterxml.jackson.databind.JsonNode; import java.util.Map; import java.util.List; public class RelaxedJsonMapperTest { @Test - public void testParseJsonWithUnquotedKeys() throws Exception { + public void testParseJsonWithFileReader() throws Exception { + // Create a temporary file reader with JSON content String json = "{name: 'John', age: 30}"; - JsonNode node = RelaxedJsonMapper.parseJson(json); + // Since parseJson only accepts FileReader, we'll test parseJsonWithTypeHandling instead + Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(json); - assertNotNull(node); - assertTrue(node.isObject()); - assertEquals("John", node.get("name").asText()); - assertEquals(30, node.get("age").asInt()); + assertTrue(result instanceof Map); + @SuppressWarnings("unchecked") + Map map = (Map) result; + assertEquals("John", map.get("name")); + assertEquals(30, map.get("age")); } - @Test - public void testParseJsonWithKeyCoercion() throws Exception { - String json = "{1: 'value1', 2: 'value2', true: 'flag', '3.14': 'pi'}"; - Map result = RelaxedJsonMapper.parseJsonWithKeyCoercion(json); + @Test + public void testParseJsonWithTypeHandling_Object() throws Exception { + // Test data from test.dsv row 4: {2:"a",1:"b"} + String json = "{2:\"a\",1:\"b\"}"; + Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(json); + + assertTrue(result instanceof Map); + @SuppressWarnings("unchecked") + Map map = (Map) result; - assertEquals("value1", result.get(1)); - assertEquals("value2", result.get(2)); - assertEquals("flag", result.get(true)); - assertEquals("pi", result.get(3.14)); + // Keys should be coerced to integers + assertEquals("a", map.get(2)); + assertEquals("b", map.get(1)); + + // String keys should not exist + assertNull(map.get("2")); + assertNull(map.get("1")); } @Test - public void testParseJsonToList() throws Exception { - String json = "[1, 'two', true, {key: 'value'}]"; - List result = RelaxedJsonMapper.parseJsonToList(json); - - assertEquals(4, result.size()); - assertEquals(1, result.get(0)); - assertEquals("two", result.get(1)); - assertEquals(true, result.get(2)); - + public void testParseJsonWithTypeHandling_Array() throws Exception { + // Test data from test.dsv row 2: ["3","1","2"] + String json = "[\"3\",\"1\",\"2\"]"; + Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(json); + + assertTrue(result instanceof List); @SuppressWarnings("unchecked") - Map obj = (Map) result.get(3); - assertEquals("value", obj.get("key")); + List list = (List) result; + + assertEquals(3, list.size()); + assertEquals("3", list.get(0)); + assertEquals("1", list.get(1)); + assertEquals("2", list.get(2)); } @Test public void testNestedObjectsInArraysWithNumericKeys() throws Exception { - String json = "[1, 2, {1: 'a', 2: 'b', true: 'flag'}]"; + // Test data from test.dsv row 3: ["1",2,{2:"a","1":"b"}] + String json = "[\"1\",2,{2:\"a\",\"1\":\"b\"}]"; Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(json); - + assertTrue(result instanceof List); @SuppressWarnings("unchecked") List list = (List) result; - + assertEquals(3, list.size()); - assertEquals(1, list.get(0)); + assertEquals("1", list.get(0)); assertEquals(2, list.get(1)); - + // Check the nested object assertTrue(list.get(2) instanceof Map); @SuppressWarnings("unchecked") Map nestedMap = (Map) list.get(2); - + // Verify that numeric keys are preserved as numbers, not strings - assertEquals("a", nestedMap.get(1)); // Key should be integer 1, not string "1" - assertEquals("b", nestedMap.get(2)); // Key should be integer 2, not string "2" - assertEquals("flag", nestedMap.get(true)); // Key should be boolean true - + assertEquals("a", nestedMap.get(2)); // Key should be integer 2 + assertEquals("b", nestedMap.get(1)); // Key should be integer 1 + // Verify that string keys would not work (they should be coerced to numbers) - assertNull(nestedMap.get("1")); // String "1" should not exist assertNull(nestedMap.get("2")); // String "2" should not exist } @Test - public void testFieldOrderPreservation() throws Exception { - // Test the specific case mentioned by user: {2:"a","1":"b"} - String json = "{2: 'a', 1: 'b', 3: 'c'}"; + public void testComplexJsonFromTestData() throws Exception { + // Test data from test.dsv row 1: Complex nested object + String json = "{\"2\": {\"createdAt\": \"1751896132789\", \"discountId\": \"3000002\", \"expiresAt\": \"1755092932789\", \"id\": \"500000\", \"updatedAt\": \"1752068932790\"}}"; Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(json); - + assertTrue(result instanceof Map); @SuppressWarnings("unchecked") Map map = (Map) result; + // Key "2" should be coerced to integer 2 + assertTrue(map.containsKey(2)); + assertFalse(map.containsKey("2")); + + Object nestedObject = map.get(2); + assertTrue(nestedObject instanceof Map); + + @SuppressWarnings("unchecked") + Map nested = (Map) nestedObject; + assertEquals("1751896132789", nested.get("createdAt")); + assertEquals("3000002", nested.get("discountId")); + assertEquals("500000", nested.get("id")); + } + + @Test + public void testFieldOrderPreservation() throws Exception { + // Test the specific case mentioned: {2:"a","1":"b"} + String json = "{2: \"a\", \"1\": \"b\", 3: \"c\"}"; + Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(json); + + assertTrue(result instanceof Map); + @SuppressWarnings("unchecked") + Map map = (Map) result; + // Verify values are correct assertEquals("a", map.get(2)); assertEquals("b", map.get(1)); assertEquals("c", map.get(3)); - + // Verify that the map preserves insertion order - // Convert to array to check order Object[] keys = map.keySet().toArray(); Object[] values = map.values().toArray(); - + // Should be in insertion order: 2, 1, 3 assertEquals(2, keys[0]); assertEquals(1, keys[1]); assertEquals(3, keys[2]); - + assertEquals("a", values[0]); assertEquals("b", values[1]); assertEquals("c", values[2]); - - System.out.println("Original JSON: " + json); - System.out.println("Parsed result keys in order: " + java.util.Arrays.toString(keys)); - System.out.println("Parsed result values in order: " + java.util.Arrays.toString(values)); } @Test - public void testParseJsonToMap() throws Exception { - String json = "{key1: 'value1', key2: 42, key3: true}"; - Map map = RelaxedJsonMapper.parseJsonToMap(json); - - assertEquals("value1", map.get("key1")); - assertEquals(42, map.get("key2")); - assertEquals(true, map.get("key3")); + public void testMixedKeyTypes() throws Exception { + String json = "{1: \"value1\", 2: \"value2\", true: \"flag\", \"3.14\": \"pi\"}"; + Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(json); + + assertTrue(result instanceof Map); + @SuppressWarnings("unchecked") + Map map = (Map) result; + + assertEquals("value1", map.get(1)); + assertEquals("value2", map.get(2)); + assertEquals("flag", map.get(true)); + assertEquals("pi", map.get(3.14)); } @Test public void testGetFromJsonNode() throws Exception { - String json = "{name: 'Test', count: 5}"; - JsonNode node = RelaxedJsonMapper.parseJson(json); + String json = "{name: \"Test\", count: 5}"; + Object parsed = RelaxedJsonMapper.parseJsonWithTypeHandling(json); + + // Since we don't have direct parseJson(String), we can test jsonNodeToObject instead + assertTrue(parsed instanceof Map); + @SuppressWarnings("unchecked") + Map map = (Map) parsed; - assertEquals("Test", RelaxedJsonMapper.getFromJsonNode(node, "name")); - assertEquals(5, RelaxedJsonMapper.getFromJsonNode(node, "count")); - assertNull(RelaxedJsonMapper.getFromJsonNode(node, "nonexistent")); + assertEquals("Test", map.get("name")); + assertEquals(5, map.get("count")); } @Test - public void testHasField() throws Exception { - String json = "{name: 'Test', count: 5}"; - JsonNode node = RelaxedJsonMapper.parseJson(json); - - assertTrue(RelaxedJsonMapper.hasField(node, "name")); - assertTrue(RelaxedJsonMapper.hasField(node, "count")); - assertFalse(RelaxedJsonMapper.hasField(node, "nonexistent")); + public void testJsonNodeToObject() throws Exception { + // Test primitive values through parseJsonWithTypeHandling + String numberJson = "123"; + Object numberResult = RelaxedJsonMapper.parseJsonWithTypeHandling(numberJson); + assertEquals(123, numberResult); + + String stringJson = "\"hello\""; + Object stringResult = RelaxedJsonMapper.parseJsonWithTypeHandling(stringJson); + assertEquals("hello", stringResult); + + String booleanJson = "true"; + Object booleanResult = RelaxedJsonMapper.parseJsonWithTypeHandling(booleanJson); + assertEquals(true, booleanResult); } @Test - public void testJsonNodeToObject() throws Exception { - String json = "{nested: {value: 123}, array: [1, 2, 3]}"; - JsonNode node = RelaxedJsonMapper.parseJson(json); + public void testJsonNodeToString() throws Exception { + // Test that we can convert parsed objects back to string representation + String json = "{key: \"value\"}"; + Object result = RelaxedJsonMapper.parseJsonWithTypeHandling(json); + + assertTrue(result instanceof Map); + @SuppressWarnings("unchecked") + Map map = (Map) result; - Object nestedObj = RelaxedJsonMapper.jsonNodeToObject(node.get("nested")); - assertTrue(nestedObj instanceof Map); + // Verify the content is parsed correctly + assertEquals("value", map.get("key")); - Object arrayObj = RelaxedJsonMapper.jsonNodeToObject(node.get("array")); - assertTrue(arrayObj instanceof List); + // The map should contain the expected key-value pair + assertTrue(map.toString().contains("key")); + assertTrue(map.toString().contains("value")); } -} \ No newline at end of file +} \ No newline at end of file