Merge pull request #39 from linkedin/dualip-lpsolver-parity

establish parity between external and internal solver

Author: aacharya-li

dualip/build.gradle

@@ -30,4 +30,4 @@ test {
 
 archivesBaseName = "${project.name}_2.12"
 
-apply from: "$rootDir/gradle/java-publication.gradle"
+apply from: "$rootDir/gradle/java-publication.gradle"

dualip/src/main/scala/com/linkedin/dualip/data/ConstrainedMatchingData.scala

@@ -0,0 +1,113 @@
+package com.linkedin.dualip.data
+
+import breeze.linalg.{SparseVector => BSV}
+import com.linkedin.dualip.projection.Projection
+import com.linkedin.dualip.util.DataFormat.DataFormat
+
+/**
+  * case class for the dual variables corresponding to local and global constraints
+  *
+  * @param lambdaLocal  : dual variables corresponding to local constraints
+  * @param lambdaGlobal : dual variables corresponding to global constraints
+  */
+case class ConstrainedMatchingDuals(lambdaLocal: Array[Double], lambdaGlobal: Array[Double])
+
+/**
+  * case class for the dual variables corresponding to local and global constraints
+  *
+  * @param lambdaLocal  : dual variables corresponding to local constraints in BSV format
+  * @param lambdaGlobal : dual variables corresponding to global constraints in BSV format
+  */
+case class ConstrainedMatchingDualsBSV(lambdaLocal: BSV[Double], lambdaGlobal: BSV[Double])
+
+/**
+  * case class for constrained matching solver parameters
+  *
+  * @param constrainedMatchingDataPath : Path of A matrix, G matrix and c vector combined in a special data block
+  * @param localBudgetPath             : Path of the budgets corresponding to local constraints
+  * @param globalBudgetPath            : Path of the budgets corresponding to global constraints
+  * @param format                      : The format of input data, e.g. avro or orc
+  * @param numOfPartitions             : number of partitions for sp
+  * @param enableHighDimOptimization   : enables high-dimensional optimization
+  * @param numLambdaPartitions         : number of partitions for the duals
+  */
+case class ConstrainedMatchingSolverParams(
+  constrainedMatchingDataPath: String,
+  localBudgetPath: String,
+  globalBudgetPath: String,
+  format: DataFormat,
+  numOfPartitions: Int,
+  enableHighDimOptimization: Boolean = false,
+  numLambdaPartitions: Option[Int] = None)
+
+
+/**
+  * case class for the A-G-c data block corresponding to the constrained matching problem
+  *
+  * @param id       : row-id
+  * @param data     : c-A-G data corresponding to a given row-id
+  * @param metadata : projection metadata
+  *
+  *                 The data field has the following structure.
+  *
+  *                 {
+  *                 "name": "data",
+  *                 "type": [
+  *                 {
+  *                 "type": "array",
+  *                 "items": [
+  *                 {
+  *                 "type": "record",
+  *                 "name": "data",
+  *                 "fields": [
+  *                 {
+  *                 "name": "colId",
+  *                 "type": "int"
+  *                 },
+  *                 {
+  *                 "name": "c",
+  *                 "type": "double"
+  *                 },
+  *                 {
+  *                 "name": "A",
+  *                 "type": "double"
+  *                 },
+  *                 {
+  *                 "name": "G",
+  *                 "type": "array",
+  *                 "items": [
+  *                 {
+  *                 "fields": [
+  *                 {
+  *                 "name": "row-Id of G",
+  *                 "type": "int"
+  *                 },
+  *                 {
+  *                 "name": "value from G matrix",
+  *                 "type": "double"
+  *                 }
+  *                 ]
+  *                 }
+  *                 ]
+  *                 }
+  *                 ]
+  *                 }
+  *                 ]
+  *                 }
+  *                 ]
+  *                 }
+  */
+case class ConstrainedMatchingData(id: String, data: Seq[(Int, Double, Double, Seq[(Int, Double)])],
+  metadata: Projection#Metadata = null)
+
+object ConstrainedMatchingData {
+  val optionalFields: Seq[String] = Seq("metadata")
+}
+
+/**
+  * case class for the budgets
+  *
+  * @param budgetLocal  : budget corresponding to local constraints
+  * @param budgetGlobal : budget corresponding to global constraints
+  */
+case class ConstrainedMatchingBudget(budgetLocal: BSV[Double], budgetGlobal: BSV[Double])

dualip/src/main/scala/com/linkedin/dualip/data/MatchingData.scala

@@ -0,0 +1,20 @@
+package com.linkedin.dualip.data
+
+import com.linkedin.dualip.projection.Projection
+
+/**
+  * Representation of the data block, used in slate generation
+  * The assumption is that spark.Dataset[MatchingData] is going to be used to store the input data,
+  * optimized for fast algorithm iterations.
+  *
+  * @param id       - id of the block (i.e. impression id).
+  * @param data     - sparse vector of tuples: (rowId, c(rowId), a(rowId))
+  *                 c(rowId) - is the objective function component of the corresponding variable
+  *                 a(rowId) - is the element of the constraint diagonal element
+  * @param metadata - features or metadata to be used for each block (string to number) mapping.
+  */
+case class MatchingData(id: String, data: Seq[(Int, Double, Double)], metadata: Projection#Metadata = null)
+
+object MatchingData {
+  val optionalFields: Seq[String] = Seq("metadata")
+}

dualip/src/main/scala/com/linkedin/dualip/data/MooData.scala

@@ -0,0 +1,27 @@
+package com.linkedin.dualip.data
+
+/**
+  * A MOO block of data. A vertical slice of design matrix, specifically the variables in the same simplex constraint sum x <= 1
+  * We need to keep this data together because we need to do a simplex projection on it.
+  *
+  * Column (variable indices in "a" and "c" are relative, that is, variable is uniquely identified by
+  * a combination of block id and internal id.
+  *
+  * internal representation is optimized for the operations that algorithm implements and data characteristics:
+  * in particular, dense constraints matrix with few rows.
+  *
+  * @param id        - unique identifier of the block, i.e. impression id for some problems
+  * @param a         - a dense constraints matrix a(row)(column)
+  * @param c         - a dense objective function vector
+  * @param problemId - unique identifier for distinguishing a specific LP problem
+  */
+case class MooData(id: Long, a: Array[Array[Double]], c: Array[Double], problemId: Long)
+
+/**
+  * A constraint block of data. A data point of constraint vector.
+  *
+  * @param row       - a specific row number of the constraint vector
+  * @param value     - the corresponding constraint value for the row
+  * @param problemId - unique identifier for distinguishing a specific LP problem
+  */
+case class ConstraintBlock(row: Int, value: Double, problemId: Long)

dualip/src/main/scala/com/linkedin/dualip/data/MultiSlateMatchingData.scala

@@ -0,0 +1,31 @@
+package com.linkedin.dualip.data
+
+import com.linkedin.dualip.projection.Projection
+
+/**
+  * case class for the Slate variables used in the non-differentiable solver
+  *
+  * @param itemId    Item ID.
+  * @param cost      Contribution to the gradient by the given item (sum over slots of a_ijk x_ijk).
+  * @param objective Contribution to the objective by the given item (sum over slots of c_ijk x_ijk).
+  * @param slots     Seq of tuples consisting of slot index and the primal values for a given item and slot, i.e.
+  *                  (k, x_ijk).
+  */
+case class SlateNonDifferentiable(itemId: Int, cost: Double, objective: Double, slots: Seq[(Int, Double)])
+
+/**
+  * case class for the A-c data block corresponding to the multi-slate matching problem
+  *
+  * @param id       Block ID.
+  * @param data     c-A data corresponding to a given block-id. The first entry of type Int in the data block is the
+  *                 item ID. The second entry is a Seq of (Int, Double, Double) triplets, each corresponding to the
+  *                 slot ID and the c and A values associated with that block ID, item ID and slot ID.
+  * @param metadata : Projection metadata.
+  *
+  */
+case class MultiSlateMatchingData(id: String, data: Seq[(Int, Seq[(Int, Double, Double)])],
+  metadata: Projection#Metadata = null)
+
+object MultiSlateMatchingData {
+  val optionalFields: Seq[String] = Seq("metadata")
+}

dualip/src/main/scala/com/linkedin/dualip/data/MultipleMatchingData.scala

@@ -0,0 +1,20 @@
+package com.linkedin.dualip.data
+
+import com.linkedin.dualip.projection.Projection
+
+/**
+  * case class for the A-c data block corresponding to the multiple matching problem
+  *
+  * @param id       : block-id
+  * @param data     : c-A data corresponding to a given block-id. The first entry of type Int in the data block is the
+  *                 item ID. The second entry of type Double is the c value associated with that (block, item). The
+  *                 third entry is a Seq of (Int, Double) pairs, each corresponding to the constraint index and A value
+  *                 for that (block, item, constraint).
+  * @param metadata : projection metadata
+  */
+case class MultipleMatchingData(id: String, data: Seq[(Int, Double, Seq[(Int, Double)])],
+  metadata: Projection#Metadata = null)
+
+object MultipleMatchingData {
+  val optionalFields: Seq[String] = Seq("metadata")
+}