Merge pull request #265 from TileDB-Inc/upgrade-to-tiledb2.16

Upgrade to tiledb2.16

Author: anastasop

.github/scripts/install_tiledb_linux.sh

@@ -1,4 +1,4 @@
 set -e -x
-curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.15.3/tiledb-linux-x86_64-2.15.3-689bea0.tar.gz \
+curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.16.0/tiledb-linux-x86_64-2.16.0-d682dd0.tar.gz \
 && sudo tar -C /usr/local -xf tiledb.tar.gz
 sudo ldconfig /usr/local/lib

.github/scripts/install_tiledb_linux_debug.sh

@@ -1,8 +1,8 @@
 set -e -x
-git clone https://github.com/TileDB-Inc/TileDB.git -b 2.15.3
+git clone https://github.com/TileDB-Inc/TileDB.git -b 2.16.0
 cd TileDB
 mkdir build && cd build
-cmake -DSANITIZER=leak -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/usr/local ..
+cmake -DTILEDB_WERROR=OFF -DTILEDB_VCPKG=ON -DSANITIZER=leak -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/usr/local ..
 make -j4
 sudo make -C tiledb install
 sudo ldconfig /usr/local/lib

.github/scripts/install_tiledb_macos.sh

@@ -1,3 +1,3 @@
 set -e -x
-curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.15.3/tiledb-macos-x86_64-2.15.3-689bea0.tar.gz \
+curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.16.0/tiledb-macos-x86_64-2.16.0-d682dd0.tar.gz \
 && sudo tar -C /usr/local -xf tiledb.tar.gz

.github/scripts/install_tiledb_source_linux.sh

@@ -1,8 +1,8 @@
 set -e -x
-git clone https://github.com/TileDB-Inc/TileDB.git -b 2.15.3
+git clone https://github.com/TileDB-Inc/TileDB.git -b 2.16.0
 cd TileDB
 mkdir build && cd build
-cmake -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local ..
+cmake -DTILEDB_WERROR=OFF -DTILEDB_VCPKG=ON -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local ..
 make -j$(nproc)
 sudo make -C tiledb install
 sudo ldconfig

.github/scripts/install_tiledb_source_macos.sh

@@ -1,7 +1,7 @@
 set -e -x
-git clone https://github.com/TileDB-Inc/TileDB.git -b 2.15.3
+git clone https://github.com/TileDB-Inc/TileDB.git -b 2.16.0
 cd TileDB
 mkdir build && cd build
-cmake -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local ..
+cmake -DTILEDB_WERROR=OFF -DTILEDB_VCPKG=ON -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local ..
 make -j$(sysctl -n hw.ncpu)
 sudo make -C tiledb install

README.md

@@ -77,6 +77,7 @@ as such the below table reference which versions are compatible.
 | 0.19.0            | 2.13.X         |
 | 0.20.0            | 2.14.X         |
 | 0.21.0            | 2.15.X         |
+| 0.22.0            | 2.16.X         |
 
 
 ## Missing Functionality

enums.go

@@ -507,6 +507,8 @@ const (
 	TILEDB_FILTER_POSITIVE_DELTA FilterType = C.TILEDB_FILTER_POSITIVE_DELTA
 	// TILEDB_FILTER_SCALE_FLOAT FILTER_SCALE_FLOAT float scaling filter.
 	TILEDB_FILTER_SCALE_FLOAT FilterType = C.TILEDB_FILTER_SCALE_FLOAT
+	// TILEDB_FILTER_DELTA Delta encoding filter.
+	TILEDB_FILTER_DELTA FilterType = C.TILEDB_FILTER_DELTA
 )
 
 // FilterOption for a given filter
@@ -562,6 +564,8 @@ const (
 	TILEDB_INCOMPLETE QueryStatus = C.TILEDB_INCOMPLETE
 	// TILEDB_UNINITIALIZED Query not initialized.
 	TILEDB_UNINITIALIZED QueryStatus = C.TILEDB_UNINITIALIZED
+	// TILEDB_INITIALIZED Query initialized.
+	TILEDB_INITIALIZED QueryStatus = C.TILEDB_INITIALIZED
 )
 
 // String returns string representation

group_test.go

@@ -276,11 +276,6 @@ func TestGroupDelete(t *testing.T) {
 		return []string{outerGroupURI, outerArrayURI, innerGroupURI, innerArrayURI}
 	}
 
-	// TileDB core versions 2.15.2, 2.15.3 and the upcoming 2.15.4
-	// are slightly different on the files they leave behind after delete
-	// To be compatible with all we check that a deleted group misses the `group.tdb` file
-	// and a delete array has an empty `__schema` dir
-
 	t.Run("recursive", func(t *testing.T) {
 		uris := setup(t)
 
@@ -304,12 +299,12 @@ func TestGroupDelete(t *testing.T) {
 		require.NoError(t, err)
 		require.False(t, exists)
 
-		dirSize, err := vfs.DirSize(uris[1] + "/__schema")
+		exists, err = vfs.IsDir(uris[1] + "/__schema")
 		require.NoError(t, err)
-		require.Equal(t, uint64(0), dirSize)
-		dirSize, err = vfs.DirSize(uris[1] + "/__schema")
+		require.False(t, exists)
+		exists, err = vfs.IsDir(uris[1] + "/__schema")
 		require.NoError(t, err)
-		require.Equal(t, uint64(0), dirSize)
+		require.False(t, exists)
 	})
 
 	t.Run("nonrecursive", func(t *testing.T) {

query_condition.go

@@ -46,6 +46,18 @@ func NewQueryConditionCombination(tdbCtx *Context, left *QueryCondition, op Quer
 	return &qc, nil
 }
 
+// NewQueryConditionNegated return the negation of the query condition. The initial condition
+// is unchanged
+func NewQueryConditionNegated(tdbCtx *Context, qc *QueryCondition) (*QueryCondition, error) {
+	nqc := QueryCondition{context: tdbCtx}
+	if ret := C.tiledb_query_condition_negate(qc.context.tiledbContext, qc.cond, &nqc.cond); ret != C.TILEDB_OK {
+		return nil, fmt.Errorf("Error allocating tiledb query condition: %s", qc.context.LastError())
+	}
+	freeOnGC(&nqc)
+
+	return &nqc, nil
+}
+
 // Free releases the internal TileDB core data that was allocated on the C heap.
 // It is automatically called when this object is garbage collected, but can be
 // called earlier to manually release memory if needed. Free is idempotent and

query_condition_test.go

@@ -47,11 +47,15 @@ func testQueryConditionInt32(t *testing.T, array *Array) {
 		name           string
 		opValue        int32
 		op             QueryConditionOp
+		negate         bool
 		expectedValues []int32
 	}{
-		{"GreaterThan1", 1, TILEDB_QUERY_CONDITION_GT, []int32{2, 3}},
-		{"LessThan3", 3, TILEDB_QUERY_CONDITION_LT, []int32{1, 2}},
-		{"EqualTo2", 2, TILEDB_QUERY_CONDITION_EQ, []int32{2}},
+		{"GreaterThan1", 1, TILEDB_QUERY_CONDITION_GT, false, []int32{2, 3}},
+		{"NotGreaterThan1", 1, TILEDB_QUERY_CONDITION_GT, true, []int32{1}},
+		{"LessThan3", 3, TILEDB_QUERY_CONDITION_LT, false, []int32{1, 2}},
+		{"NotLessThan3", 3, TILEDB_QUERY_CONDITION_LT, true, []int32{3}},
+		{"EqualTo2", 2, TILEDB_QUERY_CONDITION_EQ, false, []int32{2}},
+		{"NotEqualTo2", 2, TILEDB_QUERY_CONDITION_EQ, true, []int32{1, 3}},
 	}
 	for _, c := range a1Cases {
 		c := c
@@ -69,6 +73,11 @@ func testQueryConditionInt32(t *testing.T, array *Array) {
 			qc, err := NewQueryCondition(array.context, "a1", c.op, c.opValue)
 			require.NoError(t, err)
 
+			if c.negate {
+				qc, err = NewQueryConditionNegated(array.context, qc)
+				require.NoError(t, err)
+			}
+
 			err = query.SetQueryCondition(qc)
 			require.NoError(t, err)
 
@@ -96,6 +105,7 @@ func testQueryConditionInt32(t *testing.T, array *Array) {
 		op2                  QueryConditionOp
 		op2Value             int32
 		combinationCondition QueryConditionCombinationOp
+		negate               bool
 		expectedValues       []int32
 	}{
 		{
@@ -105,8 +115,19 @@ func testQueryConditionInt32(t *testing.T, array *Array) {
 			op2:                  TILEDB_QUERY_CONDITION_LT,
 			op2Value:             3,
 			combinationCondition: TILEDB_QUERY_CONDITION_AND,
+			negate:               false,
 			expectedValues:       []int32{2},
 		},
+		{
+			name:                 "NotGreaterThan1AndLessThan3",
+			op1:                  TILEDB_QUERY_CONDITION_GT,
+			op1Value:             1,
+			op2:                  TILEDB_QUERY_CONDITION_LT,
+			op2Value:             3,
+			combinationCondition: TILEDB_QUERY_CONDITION_AND,
+			negate:               true,
+			expectedValues:       []int32{1, 3},
+		},
 	}
 	for _, c := range a1CombinationCases {
 		c := c
@@ -130,6 +151,11 @@ func testQueryConditionInt32(t *testing.T, array *Array) {
 			qc, err := NewQueryConditionCombination(array.context, qc1, c.combinationCondition, qc2)
 			require.NoError(t, err)
 
+			if c.negate {
+				qc, err = NewQueryConditionNegated(array.context, qc)
+				require.NoError(t, err)
+			}
+
 			err = query.SetQueryCondition(qc)
 			require.NoError(t, err)