Skip to content

Commit 1a2b511

Browse files
committed
fix e2eTest regression - always serialize double values as e.g. 1.0 not 1
1 parent a9d2ee6 commit 1a2b511

File tree

1 file changed

+18
-2
lines changed

1 file changed

+18
-2
lines changed

merlin-sdk/src/main/java/gov/nasa/jpl/aerie/merlin/protocol/types/SerializedValue.java

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package gov.nasa.jpl.aerie.merlin.protocol.types;
22

33
import java.math.BigDecimal;
4+
import java.math.BigInteger;
45
import java.math.MathContext;
56
import java.util.List;
67
import java.util.Map;
@@ -132,18 +133,33 @@ public boolean equals(final Object obj) {
132133
record DoubleValue(double value) implements SerializedValue, DirectNumericValue {
133134
@Override
134135
public <T> T match(final Visitor<T> visitor) {
135-
return visitor.onNumeric(new BigDecimal(value, MathContext.DECIMAL64));
136+
return visitor.onNumeric(toBigDecimal());
136137
}
137138

138139
@Override
139140
public NumericValue asNumericValue() {
140-
return new NumericValue(new BigDecimal(value, MathContext.DECIMAL64));
141+
return new NumericValue(toBigDecimal());
141142
}
142143

143144
@Override
144145
public boolean equals(final Object obj) {
145146
return asNumericValue().equals(obj);
146147
}
148+
149+
private BigDecimal toBigDecimal() {
150+
//without MathContext.DECIMAL64 then a double assigned to from a string (or code literal) "3.14"
151+
//converts to a BigDecimal=3.140000000000000124344978758017532527446746826171875
152+
//but since a double can only represent up to 15 decimal digits when going from string -> double -> string
153+
//the nonzero values in the smaller decimal places are just an artifact of the representation
154+
//and there are unit tests that assume that string -> double -> string will be an identity op for e.g. 3.14
155+
//with MathContext.DECIMAL64 "3.14" converts to a BigDecimal=3.140000000000000
156+
var bd = new BigDecimal(value, MathContext.DECIMAL64);
157+
if (bd.scale() == 0) { //if the underlying value was actually an integer
158+
//we want to always serialize as a real number, i.e. "1.0" not "1" in JSON
159+
bd = new BigDecimal(bd.unscaledValue().multiply(BigInteger.valueOf(10)), 1); //yes scale=1 not -1
160+
}
161+
return bd;
162+
}
147163
}
148164

149165
record BooleanValue(boolean value) implements SerializedValue {

0 commit comments

Comments
 (0)