Skip to content

Commit 0942d07

Browse files
cerveadalsulak
andauthoredNov 1, 2023
#26 add create checkpoint function (#79)
* #26 add checkpoint creation function * #26: finishing bear minimum of agent side * #26: post-review improvements and refactoring * #26: extra validation, putting emphasis on types, some generics, more unit tests, refactoring & simplification * #26: functionName -> measureName, to be consistent * #26: post-review changes * #26: fixing unit test * #26: removing redundant internal Agent's Checkpoint model and some refactoring * #26: removing onlyForNumeric as it's not currently used and we might do this differently * #26: post-review changes --------- Co-authored-by: Ladislav Sulak <laco.sulak@gmail.com>
1 parent 7681d9d commit 0942d07

20 files changed

+420
-236
lines changed
 

‎LICENSE.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -199,4 +199,4 @@
199199
distributed under the License is distributed on an "AS IS" BASIS,
200200
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201201
See the License for the specific language governing permissions and
202-
limitations under the License.
202+
limitations under the License.

‎agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala

+5-13
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,10 @@ package za.co.absa.atum.agent
1818
import com.typesafe.config.{Config, ConfigFactory}
1919
import za.co.absa.atum.agent.AtumContext.AtumPartitions
2020
import za.co.absa.atum.agent.dispatcher.{ConsoleDispatcher, HttpDispatcher}
21-
import za.co.absa.atum.agent.model.Checkpoint
2221
import za.co.absa.atum.model.dto.{CheckpointDTO, PartitioningDTO}
2322

2423
/**
25-
* Place holder for the agent that communicate with the API.
24+
* Entity that communicate with the API, primarily focused on spawning Atum Context(s).
2625
*/
2726
class AtumAgent private[agent] () {
2827

@@ -36,23 +35,16 @@ class AtumAgent private[agent] () {
3635

3736
/**
3837
* Sends `CheckpointDTO` to the AtumService API
39-
* @param checkpoint
40-
*/
41-
def saveCheckpoint(checkpoint: CheckpointDTO): Unit = {
42-
dispatcher.saveCheckpoint(checkpoint)
43-
}
44-
45-
/**
46-
* Sends `Checkpoint` to the AtumService API
4738
*
48-
* @param checkpoint
39+
* @param checkpoint Already initialized Checkpoint object to store
4940
*/
50-
def saveCheckpoint(checkpoint: Checkpoint): Unit = {
51-
dispatcher.saveCheckpoint(checkpoint.toCheckpointDTO)
41+
private [agent] def saveCheckpoint(checkpoint: CheckpointDTO): Unit = {
42+
dispatcher.saveCheckpoint(checkpoint)
5243
}
5344

5445
/**
5546
* Provides an AtumContext given a `AtumPartitions` instance. Retrieves the data from AtumService API.
47+
*
5648
* @param atumPartitions
5749
* @return
5850
*/

‎agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala

+42-15
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,12 @@ package za.co.absa.atum.agent
1818

1919
import org.apache.spark.sql.DataFrame
2020
import za.co.absa.atum.agent.AtumContext.AtumPartitions
21-
import za.co.absa.atum.agent.model.{Checkpoint, Measure, Measurement, MeasuresMapper}
22-
import za.co.absa.atum.model.dto.{AtumContextDTO, PartitionDTO}
21+
import za.co.absa.atum.agent.model.Measurement.MeasurementByAtum
22+
import za.co.absa.atum.agent.model._
23+
import za.co.absa.atum.model.dto._
2324

2425
import java.time.OffsetDateTime
26+
import java.util.UUID
2527
import scala.collection.immutable.ListMap
2628

2729
/**
@@ -43,24 +45,48 @@ class AtumContext private[agent] (
4345
agent.getOrCreateAtumSubContext(atumPartitions ++ subPartitions)(this)
4446
}
4547

46-
def createCheckpoint(checkpointName: String, author: String, dataToMeasure: DataFrame) = {
47-
??? // TODO #26
48+
private def takeMeasurements(df: DataFrame): Set[MeasurementByAtum] = {
49+
measures.map { m =>
50+
val measurementResult = m.function(df)
51+
MeasurementByAtum(m, measurementResult.result, measurementResult.resultType)
52+
}
53+
}
54+
55+
def createCheckpoint(checkpointName: String, author: String, dataToMeasure: DataFrame): AtumContext = {
56+
val startTime = OffsetDateTime.now()
57+
val measurements = takeMeasurements(dataToMeasure)
58+
val endTime = OffsetDateTime.now()
59+
60+
val checkpointDTO = CheckpointDTO(
61+
id = UUID.randomUUID(),
62+
name = checkpointName,
63+
author = author,
64+
measuredByAtumAgent = true,
65+
partitioning = AtumPartitions.toSeqPartitionDTO(this.atumPartitions),
66+
processStartTime = startTime,
67+
processEndTime = Some(endTime),
68+
measurements = measurements.map(MeasurementBuilder.buildMeasurementDTO).toSeq
69+
)
70+
71+
agent.saveCheckpoint(checkpointDTO)
72+
this
4873
}
4974

50-
def createCheckpointOnProvidedData(
51-
checkpointName: String,
52-
author: String,
53-
measurements: Seq[Measurement]
54-
): Checkpoint = {
75+
def createCheckpointOnProvidedData(checkpointName: String, author: String, measurements: Seq[Measurement]): AtumContext = {
5576
val offsetDateTimeNow = OffsetDateTime.now()
56-
Checkpoint(
77+
78+
val checkpointDTO = CheckpointDTO(
79+
id = UUID.randomUUID(),
5780
name = checkpointName,
5881
author = author,
59-
atumPartitions = this.atumPartitions,
82+
partitioning = AtumPartitions.toSeqPartitionDTO(this.atumPartitions),
6083
processStartTime = offsetDateTimeNow,
6184
processEndTime = Some(offsetDateTimeNow),
62-
measurements = measurements
85+
measurements = measurements.map(MeasurementBuilder.buildMeasurementDTO)
6386
)
87+
88+
agent.saveCheckpoint(checkpointDTO)
89+
this
6490
}
6591

6692
def addAdditionalData(key: String, value: String): Unit = {
@@ -121,21 +147,22 @@ object AtumContext {
121147
new AtumContext(
122148
AtumPartitions.fromPartitioning(atumContextDTO.partitioning),
123149
agent,
124-
MeasuresMapper.mapToMeasures(atumContextDTO.measures),
150+
MeasuresBuilder.mapToMeasures(atumContextDTO.measures),
125151
atumContextDTO.additionalData.additionalData
126152
)
127153
}
128154

129155
implicit class DatasetWrapper(df: DataFrame) {
130156

131157
/**
132-
* Set a point in the pipeline to execute calculation.
158+
* Set a point in the pipeline to execute calculation and store it.
133159
* @param checkpointName The key assigned to this checkpoint
160+
* @param author Author of the checkpoint
134161
* @param atumContext Contains the calculations to be done and publish the result
135162
* @return
136163
*/
137164
def createCheckpoint(checkpointName: String, author: String)(implicit atumContext: AtumContext): DataFrame = {
138-
// todo: implement checkpoint creation
165+
atumContext.createCheckpoint(checkpointName, author, df)
139166
df
140167
}
141168

‎agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala

+9-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package za.co.absa.atum.agent.core
1818

1919
import org.apache.spark.sql.DataFrame
2020
import za.co.absa.atum.agent.core.MeasurementProcessor.MeasurementFunction
21+
import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType
2122

2223
trait MeasurementProcessor {
2324

@@ -26,6 +27,13 @@ trait MeasurementProcessor {
2627
}
2728

2829
object MeasurementProcessor {
29-
type MeasurementFunction = DataFrame => String
30+
/**
31+
* The raw result of measurement is always gonna be string, because we want to avoid some floating point issues
32+
* (overflows, consistent representation of numbers - whether they are coming from Java or Scala world, and more),
33+
* but the actual type is stored alongside the computation because we don't want to lost this information.
34+
*/
35+
final case class ResultOfMeasurement(result: String, resultType: ResultValueType.ResultValueType)
36+
37+
type MeasurementFunction = DataFrame => ResultOfMeasurement
3038

3139
}

‎agent/src/main/scala/za/co/absa/atum/agent/dispatcher/ConsoleDispatcher.scala

+1
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,5 @@ class ConsoleDispatcher extends Dispatcher with Logging {
3333
override def saveCheckpoint(checkpoint: CheckpointDTO): Unit = {
3434
println(s"Saving checkpoint to server. $checkpoint")
3535
}
36+
3637
}

‎agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala

+1
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,5 @@ class HttpDispatcher(config: Config) extends Dispatcher with Logging {
4545
.post(serverUri)
4646
.send(backend)
4747
}
48+
4849
}

‎agent/src/main/scala/za/co/absa/atum/agent/exception/UnsupportedMeasureException.scala ‎agent/src/main/scala/za/co/absa/atum/agent/exception/AtumAgentException.scala

+4-1
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,7 @@
1616

1717
package za.co.absa.atum.agent.exception
1818

19-
case class UnsupportedMeasureException(msg: String) extends Exception(msg)
19+
sealed abstract class AtumAgentException extends Exception
20+
21+
case class MeasurementProvidedException(msg: String) extends AtumAgentException
22+
case class MeasureException(msg: String) extends AtumAgentException

‎agent/src/main/scala/za/co/absa/atum/agent/exception/UnsupportedMeasureResultType.scala

-19
This file was deleted.

‎agent/src/main/scala/za/co/absa/atum/agent/model/Checkpoint.scala

-46
This file was deleted.

‎agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala

+57-33
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@ import org.apache.spark.sql.functions._
2020
import org.apache.spark.sql.types.{DecimalType, LongType, StringType}
2121
import org.apache.spark.sql.{Column, DataFrame}
2222
import za.co.absa.atum.agent.core.MeasurementProcessor
23-
import za.co.absa.atum.agent.core.MeasurementProcessor.MeasurementFunction
23+
import za.co.absa.atum.agent.core.MeasurementProcessor.{MeasurementFunction, ResultOfMeasurement}
24+
import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType
2425
import za.co.absa.spark.commons.implicits.StructTypeImplicits.StructTypeEnhancements
2526

2627
/**
@@ -32,86 +33,109 @@ sealed trait Measure extends MeasurementProcessor with MeasureType {
3233

3334
trait MeasureType {
3435
val measureName: String
35-
val onlyForNumeric: Boolean
36+
val resultValueType: ResultValueType.ResultValueType
3637
}
3738

3839
object Measure {
3940

4041
private val valueColumnName: String = "value"
4142

4243
val supportedMeasures: Seq[MeasureType] = Seq(
43-
RecordCount, DistinctRecordCount, SumOfValuesOfColumn, AbsSumOfValuesOfColumn, SumOfHashesOfColumn
44+
RecordCount,
45+
DistinctRecordCount,
46+
SumOfValuesOfColumn,
47+
AbsSumOfValuesOfColumn,
48+
SumOfHashesOfColumn
4449
)
4550
val supportedMeasureNames: Seq[String] = supportedMeasures.map(_.measureName)
4651

47-
case class RecordCount private (controlCol: String, measureName: String, onlyForNumeric: Boolean) extends Measure {
52+
case class RecordCount private (
53+
controlCol: String,
54+
measureName: String,
55+
resultValueType: ResultValueType.ResultValueType
56+
) extends Measure {
4857

4958
override def function: MeasurementFunction =
50-
(ds: DataFrame) => ds.select(col(controlCol)).count().toString
59+
(ds: DataFrame) => {
60+
val resultValue = ds.select(col(controlCol)).count().toString
61+
ResultOfMeasurement(resultValue, resultValueType)
62+
}
5163
}
5264
object RecordCount extends MeasureType {
53-
def apply(controlCol: String): RecordCount = {
54-
RecordCount(controlCol, measureName, onlyForNumeric)
55-
}
65+
def apply(controlCol: String): RecordCount = RecordCount(controlCol, measureName, resultValueType)
5666

5767
override val measureName: String = "count"
58-
override val onlyForNumeric: Boolean = false
68+
override val resultValueType: ResultValueType.ResultValueType = ResultValueType.Long
5969
}
6070

61-
case class DistinctRecordCount private (controlCol: String, measureName: String, onlyForNumeric: Boolean)
62-
extends Measure {
71+
case class DistinctRecordCount private (
72+
controlCol: String,
73+
measureName: String,
74+
resultValueType: ResultValueType.ResultValueType
75+
) extends Measure {
6376

6477
override def function: MeasurementFunction =
65-
(ds: DataFrame) => ds.select(col(controlCol)).distinct().count().toString
78+
(ds: DataFrame) => {
79+
val resultValue = ds.select(col(controlCol)).distinct().count().toString
80+
ResultOfMeasurement(resultValue, resultValueType)
81+
}
6682
}
67-
6883
object DistinctRecordCount extends MeasureType {
6984
def apply(controlCol: String): DistinctRecordCount = {
70-
DistinctRecordCount(controlCol, measureName, onlyForNumeric)
85+
DistinctRecordCount(controlCol, measureName, resultValueType)
7186
}
7287

7388
override val measureName: String = "distinctCount"
74-
override val onlyForNumeric: Boolean = false
89+
override val resultValueType: ResultValueType.ResultValueType = ResultValueType.Long
7590
}
7691

77-
case class SumOfValuesOfColumn private (controlCol: String, measureName: String, onlyForNumeric: Boolean)
78-
extends Measure {
92+
case class SumOfValuesOfColumn private (
93+
controlCol: String,
94+
measureName: String,
95+
resultValueType: ResultValueType.ResultValueType
96+
) extends Measure {
7997

8098
override def function: MeasurementFunction = (ds: DataFrame) => {
8199
val aggCol = sum(col(valueColumnName))
82-
aggregateColumn(ds, controlCol, aggCol)
100+
val resultValue = aggregateColumn(ds, controlCol, aggCol)
101+
ResultOfMeasurement(resultValue, resultValueType)
83102
}
84103
}
85-
86104
object SumOfValuesOfColumn extends MeasureType {
87105
def apply(controlCol: String): SumOfValuesOfColumn = {
88-
SumOfValuesOfColumn(controlCol, measureName, onlyForNumeric)
106+
SumOfValuesOfColumn(controlCol, measureName, resultValueType)
89107
}
90108

91109
override val measureName: String = "aggregatedTotal"
92-
override val onlyForNumeric: Boolean = true
110+
override val resultValueType: ResultValueType.ResultValueType = ResultValueType.BigDecimal
93111
}
94112

95-
case class AbsSumOfValuesOfColumn private (controlCol: String, measureName: String, onlyForNumeric: Boolean)
96-
extends Measure {
113+
case class AbsSumOfValuesOfColumn private (
114+
controlCol: String,
115+
measureName: String,
116+
resultValueType: ResultValueType.ResultValueType
117+
) extends Measure {
97118

98119
override def function: MeasurementFunction = (ds: DataFrame) => {
99120
val aggCol = sum(abs(col(valueColumnName)))
100-
aggregateColumn(ds, controlCol, aggCol)
121+
val resultValue = aggregateColumn(ds, controlCol, aggCol)
122+
ResultOfMeasurement(resultValue, resultValueType)
101123
}
102124
}
103-
104125
object AbsSumOfValuesOfColumn extends MeasureType {
105126
def apply(controlCol: String): AbsSumOfValuesOfColumn = {
106-
AbsSumOfValuesOfColumn(controlCol, measureName, onlyForNumeric)
127+
AbsSumOfValuesOfColumn(controlCol, measureName, resultValueType)
107128
}
108129

109130
override val measureName: String = "absAggregatedTotal"
110-
override val onlyForNumeric: Boolean = true
131+
override val resultValueType: ResultValueType.ResultValueType = ResultValueType.Double
111132
}
112133

113-
case class SumOfHashesOfColumn private (controlCol: String, measureName: String, onlyForNumeric: Boolean)
114-
extends Measure {
134+
case class SumOfHashesOfColumn private (
135+
controlCol: String,
136+
measureName: String,
137+
resultValueType: ResultValueType.ResultValueType
138+
) extends Measure {
115139

116140
override def function: MeasurementFunction = (ds: DataFrame) => {
117141

@@ -120,17 +144,17 @@ object Measure {
120144
.withColumn(aggregatedColumnName, crc32(col(controlCol).cast("String")))
121145
.agg(sum(col(aggregatedColumnName)))
122146
.collect()(0)(0)
123-
if (value == null) "" else value.toString
147+
val resultValue = if (value == null) "" else value.toString
148+
ResultOfMeasurement(resultValue, ResultValueType.String)
124149
}
125150
}
126-
127151
object SumOfHashesOfColumn extends MeasureType {
128152
def apply(controlCol: String): SumOfHashesOfColumn = {
129-
SumOfHashesOfColumn(controlCol, measureName, onlyForNumeric)
153+
SumOfHashesOfColumn(controlCol, measureName, resultValueType)
130154
}
131155

132156
override val measureName: String = "hashCrc32"
133-
override val onlyForNumeric: Boolean = false
157+
override val resultValueType: ResultValueType.ResultValueType = ResultValueType.String
134158
}
135159

136160
private def aggregateColumn(

‎agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala

+62-1
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,65 @@
1616

1717
package za.co.absa.atum.agent.model
1818

19-
case class Measurement(measure: Measure, result: Any)
19+
import za.co.absa.atum.agent.exception.MeasurementProvidedException
20+
import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType
21+
22+
trait Measurement {
23+
val measure: Measure
24+
val resultValue: Any
25+
val resultType: ResultValueType.ResultValueType
26+
}
27+
28+
object Measurement {
29+
30+
/**
31+
* When the application/user of Atum Agent provides actual results by himself, the type is precise and we don't need
32+
* to do any adjustments.
33+
*/
34+
case class MeasurementProvided[T](measure: Measure, resultValue: T, resultType: ResultValueType.ResultValueType)
35+
extends Measurement
36+
37+
object MeasurementProvided {
38+
39+
private def handleSpecificType[T](
40+
measure: Measure, resultValue: T, requiredType: ResultValueType.ResultValueType
41+
): MeasurementProvided[T] = {
42+
43+
val actualType = measure.resultValueType
44+
if (actualType != requiredType)
45+
throw MeasurementProvidedException(
46+
s"Type of a given provided measurement result and type that a given measure supports are not compatible! " +
47+
s"Got $actualType but should be $requiredType"
48+
)
49+
MeasurementProvided[T](measure, resultValue, requiredType)
50+
}
51+
52+
def apply[T](measure: Measure, resultValue: T): Measurement = {
53+
resultValue match {
54+
case l: Long =>
55+
handleSpecificType[Long](measure, l, ResultValueType.Long)
56+
case d: Double =>
57+
handleSpecificType[Double](measure, d, ResultValueType.Double)
58+
case bd: BigDecimal =>
59+
handleSpecificType[BigDecimal](measure, bd, ResultValueType.BigDecimal)
60+
case s: String =>
61+
handleSpecificType[String](measure, s, ResultValueType.String)
62+
63+
case unsupportedType =>
64+
val className = unsupportedType.getClass.getSimpleName
65+
throw MeasurementProvidedException(
66+
s"Unsupported type of measurement for measure ${measure.measureName}: $className " +
67+
s"for provided result: $resultValue"
68+
)
69+
}
70+
}
71+
}
72+
73+
/**
74+
* When the Atum Agent itself performs the measurements, using Spark, then in some cases some adjustments are
75+
* needed - thus we are converting the results to strings always - but we need to keep the information about
76+
* the actual type as well.
77+
*/
78+
case class MeasurementByAtum(measure: Measure, resultValue: String, resultType: ResultValueType.ResultValueType)
79+
extends Measurement
80+
}

‎agent/src/main/scala/za/co/absa/atum/agent/model/MeasurementBuilder.scala

+6-43
Original file line numberDiff line numberDiff line change
@@ -16,55 +16,18 @@
1616

1717
package za.co.absa.atum.agent.model
1818

19-
import za.co.absa.atum.agent.exception.UnsupportedMeasureResultType
2019
import za.co.absa.atum.model.dto.{MeasureDTO, MeasureResultDTO, MeasurementDTO}
21-
import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue}
20+
import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue
2221

2322
private [agent] object MeasurementBuilder {
2423

25-
def buildMeasurementDTO(measurement: Measurement): MeasurementDTO = {
24+
private [agent] def buildMeasurementDTO(measurement: Measurement): MeasurementDTO = {
2625
val measureName = measurement.measure.measureName
27-
measurement.result match {
28-
case l: Long =>
29-
buildLongMeasurement(measureName, Seq(measurement.measure.controlCol), l)
30-
case d: Double =>
31-
buildDoubleMeasureResult(measureName, Seq(measurement.measure.controlCol), d)
32-
case bd: BigDecimal =>
33-
buildBigDecimalMeasureResult(measureName, Seq(measurement.measure.controlCol), bd)
34-
case s: String =>
35-
buildStringMeasureResult(measureName, Seq(measurement.measure.controlCol), s)
36-
case unsupportedType =>
37-
val className = unsupportedType.getClass.getSimpleName
38-
throw UnsupportedMeasureResultType(s"Unsupported type of measure $measureName: $className")
39-
}
40-
}
41-
42-
private def buildLongMeasurement(functionName: String, controlCols: Seq[String], resultValue: Long): MeasurementDTO = {
43-
MeasurementDTO(
44-
MeasureDTO(functionName, controlCols),
45-
MeasureResultDTO(TypedValue(resultValue.toString, ResultValueType.Long))
46-
)
47-
}
26+
val controlCols = Seq(measurement.measure.controlCol)
27+
val measureDTO = MeasureDTO(measureName, controlCols)
4828

49-
private def buildDoubleMeasureResult(functionName: String, controlCols: Seq[String], resultValue: Double): MeasurementDTO = {
50-
MeasurementDTO(
51-
MeasureDTO(functionName, controlCols),
52-
MeasureResultDTO(TypedValue(resultValue.toString, ResultValueType.Double))
53-
)
54-
}
29+
val measureResultDTO = MeasureResultDTO(TypedValue(measurement.resultValue.toString, measurement.resultType))
5530

56-
private def buildBigDecimalMeasureResult(functionName: String, controlCols: Seq[String], resultValue: BigDecimal): MeasurementDTO = {
57-
MeasurementDTO(
58-
MeasureDTO(functionName, controlCols),
59-
MeasureResultDTO(TypedValue(resultValue.toString, ResultValueType.BigDecimal))
60-
)
31+
MeasurementDTO(measureDTO, measureResultDTO)
6132
}
62-
63-
private def buildStringMeasureResult(functionName: String, controlCols: Seq[String], resultValue: String): MeasurementDTO = {
64-
MeasurementDTO(
65-
MeasureDTO(functionName, controlCols),
66-
MeasureResultDTO(TypedValue(resultValue, ResultValueType.String))
67-
)
68-
}
69-
7033
}

‎agent/src/main/scala/za/co/absa/atum/agent/model/MeasuresMapper.scala ‎agent/src/main/scala/za/co/absa/atum/agent/model/MeasuresBuilder.scala

+5-5
Original file line numberDiff line numberDiff line change
@@ -16,27 +16,27 @@
1616

1717
package za.co.absa.atum.agent.model
1818

19-
import za.co.absa.atum.agent.exception.UnsupportedMeasureException
19+
import za.co.absa.atum.agent.exception.MeasureException
2020
import za.co.absa.atum.agent.model.Measure._
2121
import za.co.absa.atum.model.dto
2222

23-
private [agent] object MeasuresMapper {
23+
private [agent] object MeasuresBuilder {
2424

25-
def mapToMeasures(measures: Set[dto.MeasureDTO]): Set[za.co.absa.atum.agent.model.Measure] = {
25+
private [agent] def mapToMeasures(measures: Set[dto.MeasureDTO]): Set[za.co.absa.atum.agent.model.Measure] = {
2626
measures.map(createMeasure)
2727
}
2828

2929
private def createMeasure(measure: dto.MeasureDTO): za.co.absa.atum.agent.model.Measure = {
3030
val controlColumn = measure.controlColumns.head
3131

32-
measure.functionName match {
32+
measure.measureName match {
3333
case RecordCount.measureName => RecordCount(controlColumn)
3434
case DistinctRecordCount.measureName => DistinctRecordCount(controlColumn)
3535
case SumOfValuesOfColumn.measureName => SumOfValuesOfColumn(controlColumn)
3636
case AbsSumOfValuesOfColumn.measureName => AbsSumOfValuesOfColumn(controlColumn)
3737
case SumOfHashesOfColumn.measureName => SumOfHashesOfColumn(controlColumn)
3838
case unsupportedMeasure =>
39-
throw UnsupportedMeasureException(
39+
throw MeasureException(
4040
s"Measure not supported: $unsupportedMeasure. Supported measures are: ${Measure.supportedMeasureNames}"
4141
)
4242
}

‎agent/src/test/scala/za/co/absa/atum/agent/AtumContextTest.scala

+106-12
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,19 @@
1616

1717
package za.co.absa.atum.agent
1818

19+
import org.apache.spark.sql.types.DoubleType
20+
import org.apache.spark.sql.{Row, SparkSession}
21+
import org.mockito.Mockito.{mock, times, verify}
22+
import org.mockito.ArgumentCaptor
1923
import org.scalatest.flatspec.AnyFlatSpec
2024
import org.scalatest.matchers.should.Matchers
2125
import za.co.absa.atum.agent.AtumContext.AtumPartitions
2226
import za.co.absa.atum.agent.model.Measure.{RecordCount, SumOfValuesOfColumn}
23-
import za.co.absa.atum.agent.model.Measurement
27+
import za.co.absa.atum.agent.model.MeasurementBuilder
28+
import za.co.absa.atum.agent.model.Measurement.MeasurementProvided
29+
import org.apache.spark.sql.types.{StringType, StructField, StructType}
30+
import za.co.absa.atum.model.dto.CheckpointDTO
31+
import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType
2432

2533
class AtumContextTest extends AnyFlatSpec with Matchers {
2634

@@ -46,7 +54,6 @@ class AtumContextTest extends AnyFlatSpec with Matchers {
4654
)
4755

4856
assert(atumContextWithTwoDistinctRecordCount.currentMeasures.size == 3)
49-
5057
}
5158

5259
"withMeasureRemoved" should "remove a measure if exists" in {
@@ -64,25 +71,112 @@ class AtumContextTest extends AnyFlatSpec with Matchers {
6471
assert(atumContextRemoved.currentMeasures.head == RecordCount("other"))
6572
}
6673

74+
"createCheckpoint" should "take measurements and create a Checkpoint" in {
75+
val mockAgent = mock(classOf[AtumAgent])
76+
val atumPartitions = AtumPartitions("foo2" -> "bar")
77+
78+
val atumContext = new AtumContext(atumPartitions, mockAgent)
79+
.addMeasure(RecordCount("letter"))
80+
81+
val spark = SparkSession.builder
82+
.master("local")
83+
.config("spark.driver.host", "localhost")
84+
.config("spark.ui.enabled", "false")
85+
.getOrCreate()
86+
87+
import spark.implicits._
88+
89+
val rdd = spark.sparkContext.parallelize(Seq("A", "B", "C"))
90+
val df = rdd.toDF("letter")
91+
92+
atumContext.createCheckpoint("testCheckpoint", "Hans", df)
93+
94+
val argument = ArgumentCaptor.forClass(classOf[CheckpointDTO])
95+
verify(mockAgent).saveCheckpoint(argument.capture())
96+
97+
assert(argument.getValue.name == "testCheckpoint")
98+
assert(argument.getValue.author == "Hans")
99+
assert(argument.getValue.partitioning == AtumPartitions.toSeqPartitionDTO(atumPartitions))
100+
assert(argument.getValue.measurements.head.result.mainValue.value == "3")
101+
assert(argument.getValue.measurements.head.result.mainValue.valueType == ResultValueType.Long)
102+
}
103+
67104
"createCheckpointOnProvidedData" should "create a Checkpoint on provided data" in {
68-
val atumAgent = new AtumAgent
105+
val mockAgent = mock(classOf[AtumAgent])
69106
val atumPartitions = AtumPartitions("key" -> "value")
70-
val atumContext = atumAgent.getOrCreateAtumContext(atumPartitions)
107+
val atumContext: AtumContext = new AtumContext(atumPartitions, mockAgent)
71108

72-
val measurements = Seq(Measurement(RecordCount("col"), "1"), Measurement(SumOfValuesOfColumn("col"), 1))
109+
val measurements = Seq(
110+
MeasurementProvided(RecordCount("col"), 1L),
111+
MeasurementProvided(SumOfValuesOfColumn("col"), BigDecimal(1))
112+
)
73113

74-
val checkpoint = atumContext.createCheckpointOnProvidedData(
114+
atumContext.createCheckpointOnProvidedData(
75115
checkpointName = "name",
76116
author = "author",
77117
measurements = measurements
78118
)
79119

80-
assert(checkpoint.name == "name")
81-
assert(checkpoint.author == "author")
82-
assert(!checkpoint.measuredByAtumAgent)
83-
assert(checkpoint.atumPartitions == atumPartitions)
84-
assert(checkpoint.processStartTime == checkpoint.processEndTime.get)
85-
assert(checkpoint.measurements == measurements)
120+
val argument = ArgumentCaptor.forClass(classOf[CheckpointDTO])
121+
verify(mockAgent).saveCheckpoint(argument.capture())
122+
123+
assert(argument.getValue.name == "name")
124+
assert(argument.getValue.author == "author")
125+
assert(!argument.getValue.measuredByAtumAgent)
126+
assert(argument.getValue.partitioning == AtumPartitions.toSeqPartitionDTO(atumPartitions))
127+
assert(argument.getValue.processStartTime == argument.getValue.processEndTime.get)
128+
assert(argument.getValue.measurements == measurements.map(MeasurementBuilder.buildMeasurementDTO))
129+
}
130+
131+
"createCheckpoint" should "take measurements and create a Checkpoint, multiple measure changes" in {
132+
val mockAgent = mock(classOf[AtumAgent])
133+
val atumPartitions = AtumPartitions("foo2" -> "bar")
134+
implicit val atumContext: AtumContext = new AtumContext(atumPartitions, mockAgent)
135+
.addMeasure(RecordCount("notImportantColumn"))
136+
137+
val spark = SparkSession.builder
138+
.master("local")
139+
.config("spark.driver.host", "localhost")
140+
.config("spark.ui.enabled", "false")
141+
.getOrCreate()
142+
143+
val rdd = spark.sparkContext.parallelize(
144+
Seq(
145+
Row("A", 8.0),
146+
Row("B", 2.9),
147+
Row("C", 9.1),
148+
Row("D", 2.5)
149+
)
150+
)
151+
val schema = new StructType()
152+
.add(StructField("notImportantColumn", StringType))
153+
.add(StructField("columnForSum", DoubleType))
154+
155+
import AtumContext._
156+
157+
val df = spark.createDataFrame(rdd, schema)
158+
.createCheckpoint("checkPointNameCount", "authorOfCount")
159+
160+
val argumentFirst = ArgumentCaptor.forClass(classOf[CheckpointDTO])
161+
verify(mockAgent, times(1)).saveCheckpoint(argumentFirst.capture())
162+
163+
assert(argumentFirst.getValue.name == "checkPointNameCount")
164+
assert(argumentFirst.getValue.author == "authorOfCount")
165+
assert(argumentFirst.getValue.partitioning == AtumPartitions.toSeqPartitionDTO(atumPartitions))
166+
assert(argumentFirst.getValue.measurements.head.result.mainValue.value == "4")
167+
assert(argumentFirst.getValue.measurements.head.result.mainValue.valueType == ResultValueType.Long)
168+
169+
atumContext.addMeasure(SumOfValuesOfColumn("columnForSum"))
170+
df.createCheckpoint("checkPointNameSum", "authorOfSum")
171+
172+
val argumentSecond = ArgumentCaptor.forClass(classOf[CheckpointDTO])
173+
verify(mockAgent, times(2)).saveCheckpoint(argumentSecond.capture())
174+
175+
assert(argumentSecond.getValue.name == "checkPointNameSum")
176+
assert(argumentSecond.getValue.author == "authorOfSum")
177+
assert(argumentSecond.getValue.partitioning == AtumPartitions.toSeqPartitionDTO(atumPartitions))
178+
assert(argumentSecond.getValue.measurements.tail.head.result.mainValue.value == "22.5")
179+
assert(argumentSecond.getValue.measurements.tail.head.result.mainValue.valueType == ResultValueType.BigDecimal)
86180
}
87181

88182
"addAdditionalData" should "add key/value pair to map for additional data" in {

‎agent/src/test/scala/za/co/absa/atum/agent/model/MeasureTest.scala

+27-12
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import org.scalatest.matchers.should.Matchers
2121
import za.co.absa.atum.agent.AtumAgent
2222
import za.co.absa.atum.agent.AtumContext.{AtumPartitions, DatasetWrapper}
2323
import za.co.absa.atum.agent.model.Measure._
24+
import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType
2425
import za.co.absa.spark.commons.test.SparkTestBase
2526

2627
class MeasureTest extends AnyFlatSpec with Matchers with SparkTestBase { self =>
@@ -36,11 +37,14 @@ class MeasureTest extends AnyFlatSpec with Matchers with SparkTestBase { self =>
3637
val sumOfHashes: Measure = SumOfHashesOfColumn(controlCol = "id")
3738

3839
// AtumContext contains `Measurement`
39-
val atumContextInstanceWithRecordCount = AtumAgent.getOrCreateAtumContext(AtumPartitions("foo"->"bar"))
40+
val atumContextInstanceWithRecordCount = AtumAgent
41+
.getOrCreateAtumContext(AtumPartitions("foo"->"bar"))
4042
.addMeasure(measureIds)
41-
val atumContextWithSalaryAbsMeasure = atumContextInstanceWithRecordCount.subPartitionContext(AtumPartitions("sub"->"partition"))
43+
val atumContextWithSalaryAbsMeasure = atumContextInstanceWithRecordCount
44+
.subPartitionContext(AtumPartitions("sub"->"partition"))
4245
.addMeasure(salaryAbsSum)
43-
val atumContextWithNameHashSum = atumContextInstanceWithRecordCount.subPartitionContext(AtumPartitions("another"->"partition"))
46+
val atumContextWithNameHashSum = atumContextInstanceWithRecordCount
47+
.subPartitionContext(AtumPartitions("another"->"partition"))
4448
.addMeasure(sumOfHashes)
4549

4650
// Pipeline
@@ -56,8 +60,7 @@ class MeasureTest extends AnyFlatSpec with Matchers with SparkTestBase { self =>
5660
.option("header", "true")
5761
.load("agent/src/test/resources/random-dataset/persons-enriched.csv")
5862
.createCheckpoint("name3", "author")(
59-
atumContextWithSalaryAbsMeasure
60-
.removeMeasure(salaryAbsSum)
63+
atumContextWithSalaryAbsMeasure.removeMeasure(salaryAbsSum)
6164
)
6265

6366
val dfFull = dfPersons
@@ -80,14 +83,26 @@ class MeasureTest extends AnyFlatSpec with Matchers with SparkTestBase { self =>
8083
.removeMeasure(salaryAbsSum)
8184
)
8285

83-
// Assertions
84-
assert(measureIds.function(dfPersons) == "1000")
85-
assert(measureIds.function(dfFull) == "1000")
86-
assert(salaryAbsSum.function(dfFull) == "2987144")
87-
assert(sumOfHashes.function(dfFull) == "2044144307532")
88-
assert(salarySum.function(dfExtraPerson) == "2986144")
89-
assert(salarySum.function(dfFull) == "2987144")
86+
val dfPersonCntResult = measureIds.function(dfPersons)
87+
val dfFullCntResult = measureIds.function(dfFull)
88+
val dfFullSalaryAbsSumResult = salaryAbsSum.function(dfFull)
89+
val dfFullHashResult = sumOfHashes.function(dfFull)
90+
val dfExtraPersonSalarySumResult = salarySum.function(dfExtraPerson)
91+
val dfFullSalarySumResult = salarySum.function(dfFull)
9092

93+
// Assertions
94+
assert(dfPersonCntResult.result == "1000")
95+
assert(dfPersonCntResult.resultType == ResultValueType.Long)
96+
assert(dfFullCntResult.result == "1000")
97+
assert(dfFullCntResult.resultType == ResultValueType.Long)
98+
assert(dfFullSalaryAbsSumResult.result == "2987144")
99+
assert(dfFullSalaryAbsSumResult.resultType == ResultValueType.Double)
100+
assert(dfFullHashResult.result == "2044144307532")
101+
assert(dfFullHashResult.resultType == ResultValueType.String)
102+
assert(dfExtraPersonSalarySumResult.result == "2986144")
103+
assert(dfExtraPersonSalarySumResult.resultType == ResultValueType.BigDecimal)
104+
assert(dfFullSalarySumResult.result == "2987144")
105+
assert(dfFullSalarySumResult.resultType == ResultValueType.BigDecimal)
91106
}
92107

93108
}

‎agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementBuilderTest.scala

+19-20
Original file line numberDiff line numberDiff line change
@@ -17,63 +17,62 @@
1717
package za.co.absa.atum.agent.model
1818

1919
import org.scalatest.flatspec.AnyFlatSpec
20-
import za.co.absa.atum.agent.exception.UnsupportedMeasureResultType
2120
import za.co.absa.atum.agent.model.Measure.SumOfValuesOfColumn
21+
import za.co.absa.atum.agent.model.Measurement.{MeasurementProvided, MeasurementByAtum}
2222
import za.co.absa.atum.model.dto.{MeasureDTO, MeasureResultDTO}
2323
import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue}
2424

2525
class MeasurementBuilderTest extends AnyFlatSpec {
2626

27-
"buildMeasurementDTO" should "build expected MeasurementDTO for Long type of result value" in {
27+
"buildMeasurementDTO" should "build MeasurementDTO for Long type of result value when Measurement provided" in {
2828
val measure = SumOfValuesOfColumn("col")
29-
val measurement = Measurement(measure, 1L)
29+
val measurement = MeasurementProvided(measure, BigDecimal(1))
3030
val measurementDTO = MeasurementBuilder.buildMeasurementDTO(measurement)
3131

3232
val expectedMeasureDTO = MeasureDTO("aggregatedTotal", Seq("col"))
3333

3434
val expectedMeasureResultDTO = MeasureResultDTO(
35-
TypedValue("1", ResultValueType.Long)
35+
TypedValue("1", ResultValueType.BigDecimal)
3636
)
3737

3838
assert(measurementDTO.measure == expectedMeasureDTO)
3939
assert(measurementDTO.result == expectedMeasureResultDTO)
4040
}
4141

42-
"buildMeasurementDTO" should "build MeasurementDTO with expected TypedValue for Double type of result value" in {
42+
"buildMeasurementDTO" should "build MeasurementDTO for BigDecimal type of result value when Measurement provided" in {
4343
val measure = SumOfValuesOfColumn("col")
44-
val measurement = Measurement(measure, 3.14)
44+
val measurement = MeasurementProvided(measure, BigDecimal(3.14))
4545
val measurementDTO = MeasurementBuilder.buildMeasurementDTO(measurement)
4646

47-
val expectedTypedValue = TypedValue("3.14", ResultValueType.Double)
47+
val expectedTypedValue = TypedValue("3.14", ResultValueType.BigDecimal)
4848

4949
assert(measurementDTO.result.mainValue == expectedTypedValue)
5050
}
5151

52-
"buildMeasurementDTO" should "build MeasurementDTO with expected TypedValue for BigDecimal type of result value" in {
52+
"buildMeasurementDTO" should "not build MeasurementDTO for incompatible String type of result value when Measurement provided" in {
5353
val measure = SumOfValuesOfColumn("col")
54-
val measurement = Measurement(measure, BigDecimal(3.14))
54+
val measurement = MeasurementByAtum(measure, "stringValue", ResultValueType.String)
55+
5556
val measurementDTO = MeasurementBuilder.buildMeasurementDTO(measurement)
5657

57-
val expectedTypedValue = TypedValue("3.14", ResultValueType.BigDecimal)
58+
val expectedTypedValue = TypedValue("stringValue", ResultValueType.String)
5859

5960
assert(measurementDTO.result.mainValue == expectedTypedValue)
6061
}
6162

62-
"buildMeasurementDTO" should "build MeasurementDTO with expected TypedValue for String type of result value" in {
63+
"buildMeasurementDTO" should "build MeasurementDTO for BigDecimal type of result value when measured by Agent" in {
6364
val measure = SumOfValuesOfColumn("col")
64-
val measurement = Measurement(measure, "stringValue")
65+
val measurement = MeasurementByAtum(measure, "1", ResultValueType.BigDecimal)
6566
val measurementDTO = MeasurementBuilder.buildMeasurementDTO(measurement)
6667

67-
val expectedTypedValue = TypedValue("stringValue", ResultValueType.String)
68-
69-
assert(measurementDTO.result.mainValue == expectedTypedValue)
70-
}
68+
val expectedMeasureDTO = MeasureDTO("aggregatedTotal", Seq("col"))
7169

72-
"buildMeasurementDTO" should "throw exception for unsupported result value type" in {
73-
val measure = SumOfValuesOfColumn("col")
74-
val measurement = Measurement(measure, 1)
70+
val expectedMeasureResultDTO = MeasureResultDTO(
71+
TypedValue("1", ResultValueType.BigDecimal)
72+
)
7573

76-
assertThrows[UnsupportedMeasureResultType](MeasurementBuilder.buildMeasurementDTO(measurement))
74+
assert(measurementDTO.measure == expectedMeasureDTO)
75+
assert(measurementDTO.result == expectedMeasureResultDTO)
7776
}
7877

7978
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/*
2+
* Copyright 2021 ABSA Group Limited
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package za.co.absa.atum.agent.model
18+
19+
import org.scalatest.flatspec.AnyFlatSpec
20+
import org.scalatest.matchers.should.Matchers
21+
import za.co.absa.atum.agent.exception.MeasurementProvidedException
22+
import za.co.absa.atum.agent.model.Measure._
23+
import za.co.absa.atum.agent.model.Measurement.MeasurementProvided
24+
import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType
25+
import za.co.absa.spark.commons.test.SparkTestBase
26+
27+
class MeasurementTest extends AnyFlatSpec with Matchers with SparkTestBase { self =>
28+
29+
"MeasurementProvided" should "be able to be converted to MeasurementProvided object when the result is Double" in {
30+
val measure = AbsSumOfValuesOfColumn("col")
31+
val actualMeasurement = MeasurementProvided(measure, 1.0)
32+
33+
assert(actualMeasurement.resultValue == 1.0)
34+
assert(actualMeasurement.resultType == ResultValueType.Double)
35+
}
36+
37+
"MeasurementProvided" should "throw exception for unsupported result value - BigDecimal instead of Double" in {
38+
val measure = AbsSumOfValuesOfColumn("col")
39+
assertThrows[MeasurementProvidedException](MeasurementProvided(measure, BigDecimal(1.0)))
40+
}
41+
42+
"MeasurementProvided" should "throw exception for unsupported result value type in general (scalar)" in {
43+
val measure = SumOfValuesOfColumn("col")
44+
assertThrows[MeasurementProvidedException](MeasurementProvided(measure, 1))
45+
}
46+
47+
"MeasurementProvided" should "throw exception for unsupported result value type in general (composite)" in {
48+
val measure = SumOfHashesOfColumn("col")
49+
assertThrows[MeasurementProvidedException](MeasurementProvided(measure, Map(1 -> "no-go")))
50+
}
51+
52+
"MeasurementProvided" should "throw exception for unsupported result value type for a given Measure" in {
53+
val measure = DistinctRecordCount("col")
54+
assertThrows[MeasurementProvidedException](MeasurementProvided(measure, "1"))
55+
}
56+
57+
"MeasurementProvided" should "throw exception for unsupported (slightly different FPN) result value type for a given Measure" in {
58+
val measure = SumOfValuesOfColumn("col")
59+
assertThrows[MeasurementProvidedException](MeasurementProvided(measure, 1.0))
60+
}
61+
}

‎agent/src/test/scala/za/co/absa/atum/agent/model/MeasuresMapperTest.scala ‎agent/src/test/scala/za/co/absa/atum/agent/model/MeasuresBuilderTest.scala

+5-5
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@
1717
package za.co.absa.atum.agent.model
1818

1919
import org.scalatest.flatspec.AnyFlatSpecLike
20-
import za.co.absa.atum.agent.exception.UnsupportedMeasureException
21-
import za.co.absa.atum.agent.model.Measure.{AbsSumOfValuesOfColumn, DistinctRecordCount, RecordCount, SumOfHashesOfColumn, SumOfValuesOfColumn}
20+
import za.co.absa.atum.agent.exception.MeasureException
21+
import za.co.absa.atum.agent.model.Measure._
2222
import za.co.absa.atum.model.dto.MeasureDTO
2323

24-
class MeasuresMapperTest extends AnyFlatSpecLike {
24+
class MeasuresBuilderTest extends AnyFlatSpecLike {
2525

2626
"mapToMeasures" should "map MeasureDTO into Measure for supported measures" in {
2727
val supportedMeasures = Set(
@@ -40,7 +40,7 @@ class MeasuresMapperTest extends AnyFlatSpecLike {
4040
SumOfHashesOfColumn("hashCrc32Col")
4141
)
4242

43-
val actualMeasures = MeasuresMapper.mapToMeasures(supportedMeasures)
43+
val actualMeasures = MeasuresBuilder.mapToMeasures(supportedMeasures)
4444

4545
assert(expectedMeasures == actualMeasures)
4646
}
@@ -50,7 +50,7 @@ class MeasuresMapperTest extends AnyFlatSpecLike {
5050
MeasureDTO("unsupportedMeasure", Seq("col"))
5151
)
5252

53-
assertThrows[UnsupportedMeasureException](MeasuresMapper.mapToMeasures(unsupportedMeasure))
53+
assertThrows[MeasureException](MeasuresBuilder.mapToMeasures(unsupportedMeasure))
5454
}
5555

5656
}

‎model/src/main/scala/za/co/absa/atum/model/dto/MeasureDTO.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,6 @@
1717
package za.co.absa.atum.model.dto
1818

1919
case class MeasureDTO(
20-
functionName: String,
20+
measureName: String,
2121
controlColumns: Seq[String]
2222
)

‎model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsTest.scala

+8-8
Original file line numberDiff line numberDiff line change
@@ -82,14 +82,14 @@ class SerializationUtilsTest extends AnyFlatSpecLike {
8282
measures = seqMeasureDTO
8383
)
8484

85-
val expectedAdditionalDataJson = "{\"partitioning\":[{\"key\":\"key\",\"value\":\"val\"}],\"measures\":[{\"functionName\":\"count\",\"controlColumns\":[\"col\"]}],\"additionalData\":{\"additionalData\":{}}}"
85+
val expectedAdditionalDataJson = "{\"partitioning\":[{\"key\":\"key\",\"value\":\"val\"}],\"measures\":[{\"measureName\":\"count\",\"controlColumns\":[\"col\"]}],\"additionalData\":{\"additionalData\":{}}}"
8686
val actualAdditionalDataJson = SerializationUtils.asJson(atumContextDTO)
8787

8888
assert(expectedAdditionalDataJson == actualAdditionalDataJson)
8989
}
9090

9191
"fromJson" should "deserialize AtumContextDTO from json string" in {
92-
val atumContextDTOJson = "{\"partitioning\":[{\"key\":\"key\",\"value\":\"val\"}],\"measures\":[{\"functionName\":\"count\",\"controlColumns\":[\"col\"]}],\"additionalData\":{\"additionalData\":{}}}"
92+
val atumContextDTOJson = "{\"partitioning\":[{\"key\":\"key\",\"value\":\"val\"}],\"measures\":[{\"measureName\":\"count\",\"controlColumns\":[\"col\"]}],\"additionalData\":{\"additionalData\":{}}}"
9393

9494
val seqPartitionDTO = Seq(PartitionDTO("key", "val"))
9595
val seqMeasureDTO = Set(MeasureDTO("count", Seq("col")))
@@ -150,7 +150,7 @@ class SerializationUtilsTest extends AnyFlatSpecLike {
150150
measurements = seqMeasurementDTO
151151
)
152152

153-
val expectedCheckpointDTOJson = "{\"id\":\"" + uuid + "\",\"name\":\"checkpoint\",\"author\":\"author\",\"measuredByAtumAgent\":true,\"partitioning\":[{\"key\":\"key\",\"value\":\"val\"}],\"processStartTime\":\"2023-10-24 10:20:59.005000+02\",\"processEndTime\":\"2023-10-24 10:20:59.005000+02\",\"measurements\":[{\"measure\":{\"functionName\":\"count\",\"controlColumns\":[\"col\"]},\"result\":{\"mainValue\":{\"value\":\"1\",\"valueType\":\"Long\"},\"supportValues\":{}}}]}"
153+
val expectedCheckpointDTOJson = "{\"id\":\"" + uuid + "\",\"name\":\"checkpoint\",\"author\":\"author\",\"measuredByAtumAgent\":true,\"partitioning\":[{\"key\":\"key\",\"value\":\"val\"}],\"processStartTime\":\"2023-10-24 10:20:59.005000+02\",\"processEndTime\":\"2023-10-24 10:20:59.005000+02\",\"measurements\":[{\"measure\":{\"measureName\":\"count\",\"controlColumns\":[\"col\"]},\"result\":{\"mainValue\":{\"value\":\"1\",\"valueType\":\"Long\"},\"supportValues\":{}}}]}"
154154
val actualCheckpointDTOJson = SerializationUtils.asJson(checkpointDTO)
155155

156156
assert(expectedCheckpointDTOJson == actualCheckpointDTOJson)
@@ -161,7 +161,7 @@ class SerializationUtilsTest extends AnyFlatSpecLike {
161161
val seqPartitionDTO = Seq(PartitionDTO("key", "val"))
162162
val timeWithZone = OffsetDateTime.of(2023, 10, 24, 10, 20, 59, 5000000, ZoneOffset.ofHours(2))
163163

164-
val checkpointDTOJson = "{\"id\":\"" + uuid + "\",\"name\":\"checkpoint\",\"author\":\"author\",\"measuredByAtumAgent\":true,\"partitioning\":[{\"key\":\"key\",\"value\":\"val\"}],\"processStartTime\":\"2023-10-24 10:20:59.005000+02\",\"processEndTime\":\"2023-10-24 10:20:59.005000+02\",\"measurements\":[{\"measure\":{\"functionName\":\"count\",\"controlColumns\":[\"col\"]},\"result\":{\"mainValue\":{\"value\":\"1\",\"valueType\":\"Long\"},\"supportValues\":{}}}]}"
164+
val checkpointDTOJson = "{\"id\":\"" + uuid + "\",\"name\":\"checkpoint\",\"author\":\"author\",\"measuredByAtumAgent\":true,\"partitioning\":[{\"key\":\"key\",\"value\":\"val\"}],\"processStartTime\":\"2023-10-24 10:20:59.005000+02\",\"processEndTime\":\"2023-10-24 10:20:59.005000+02\",\"measurements\":[{\"measure\":{\"measureName\":\"count\",\"controlColumns\":[\"col\"]},\"result\":{\"mainValue\":{\"value\":\"1\",\"valueType\":\"Long\"},\"supportValues\":{}}}]}"
165165

166166
val seqMeasurementDTO = Seq(
167167
MeasurementDTO(
@@ -191,14 +191,14 @@ class SerializationUtilsTest extends AnyFlatSpecLike {
191191
"asJson" should "serialize MeasureDTO into json string" in {
192192
val measureDTO = MeasureDTO("count", Seq("col"))
193193

194-
val expectedMeasureDTOJson = "{\"functionName\":\"count\",\"controlColumns\":[\"col\"]}"
194+
val expectedMeasureDTOJson = "{\"measureName\":\"count\",\"controlColumns\":[\"col\"]}"
195195
val actualMeasureDTOJson = SerializationUtils.asJson(measureDTO)
196196

197197
assert(expectedMeasureDTOJson == actualMeasureDTOJson)
198198
}
199199

200200
"fromJson" should "deserialize MeasureDTO from json string" in {
201-
val measureDTOJson = "{\"functionName\":\"count\",\"controlColumns\":[\"col\"]}"
201+
val measureDTOJson = "{\"measureName\":\"count\",\"controlColumns\":[\"col\"]}"
202202

203203
val expectedMeasureDTO = MeasureDTO("count", Seq("col"))
204204
val actualMeasureDTO = SerializationUtils.fromJson[MeasureDTO](measureDTOJson)
@@ -213,14 +213,14 @@ class SerializationUtilsTest extends AnyFlatSpecLike {
213213

214214
val measurementDTO = MeasurementDTO(measureDTO, measureResultDTO)
215215

216-
val expectedMeasurementDTOJson = "{\"measure\":{\"functionName\":\"count\",\"controlColumns\":[\"col\"]},\"result\":{\"mainValue\":{\"value\":\"1\",\"valueType\":\"Long\"},\"supportValues\":{}}}"
216+
val expectedMeasurementDTOJson = "{\"measure\":{\"measureName\":\"count\",\"controlColumns\":[\"col\"]},\"result\":{\"mainValue\":{\"value\":\"1\",\"valueType\":\"Long\"},\"supportValues\":{}}}"
217217
val actualMeasurementDTOJson = SerializationUtils.asJson(measurementDTO)
218218

219219
assert(expectedMeasurementDTOJson == actualMeasurementDTOJson)
220220
}
221221

222222
"fromJson" should "deserialize MeasurementDTO from json string" in {
223-
val measurementDTOJson = "{\"measure\":{\"functionName\":\"count\",\"controlColumns\":[\"col\"]},\"result\":{\"mainValue\":{\"value\":\"1\",\"valueType\":\"Long\"},\"supportValues\":{}}}"
223+
val measurementDTOJson = "{\"measure\":{\"measureName\":\"count\",\"controlColumns\":[\"col\"]},\"result\":{\"mainValue\":{\"value\":\"1\",\"valueType\":\"Long\"},\"supportValues\":{}}}"
224224

225225
val measureDTO = MeasureDTO("count", Seq("col"))
226226
val measureResultDTO = MeasureResultDTO(mainValue = TypedValue("1", ResultValueType.Long))

0 commit comments

Comments
 (0)
Please sign in to comment.