PG17 compatibility: Fix Test Failure in dml_recursive

.github/workflows/build_and_test.yml

@@ -27,11 +27,12 @@ jobs:
       style_checker_image_name: "ghcr.io/citusdata/stylechecker"
       style_checker_tools_version: "0.8.18"
       sql_snapshot_pg_version: "16.3"
-      image_suffix: "-v13fd57c"
+      image_suffix: "-dev-e5b0348"
       pg14_version: '{ "major": "14", "full": "14.12" }'
       pg15_version: '{ "major": "15", "full": "15.7" }'
       pg16_version: '{ "major": "16", "full": "16.3" }'
-      upgrade_pg_versions: "14.12-15.7-16.3"
+      pg17_version: '{ "major": "17", "full": "17.0" }'
+      upgrade_pg_versions: "14.12-15.7-16.3-17.0"
     steps:
       # Since GHA jobs needs at least one step we use a noop step here.
       - name: Set up parameters
@@ -108,6 +109,7 @@ jobs:
           - ${{ needs.params.outputs.pg14_version }}
           - ${{ needs.params.outputs.pg15_version }}
           - ${{ needs.params.outputs.pg16_version }}
+          - ${{ needs.params.outputs.pg17_version }}
     runs-on: ubuntu-20.04
     container:
       image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}"
@@ -139,6 +141,7 @@ jobs:
           - ${{ needs.params.outputs.pg14_version }}
           - ${{ needs.params.outputs.pg15_version }}
           - ${{ needs.params.outputs.pg16_version }}
+          - ${{ needs.params.outputs.pg17_version }}
         make:
           - check-split
           - check-multi
@@ -168,6 +171,10 @@ jobs:
             pg_version: ${{ needs.params.outputs.pg16_version }}
             suite: regress
             image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: check-failure
+            pg_version: ${{ needs.params.outputs.pg17_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
           - make: check-enterprise-failure
             pg_version: ${{ needs.params.outputs.pg14_version }}
             suite: regress
@@ -180,6 +187,10 @@ jobs:
             pg_version: ${{ needs.params.outputs.pg16_version }}
             suite: regress
             image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: check-enterprise-failure
+            pg_version: ${{ needs.params.outputs.pg17_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
           - make: check-pytest
             pg_version: ${{ needs.params.outputs.pg14_version }}
             suite: regress
@@ -192,6 +203,10 @@ jobs:
             pg_version: ${{ needs.params.outputs.pg16_version }}
             suite: regress
             image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: check-pytest
+            pg_version: ${{ needs.params.outputs.pg17_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
           - make: installcheck
             suite: cdc
             image_name: ${{ needs.params.outputs.test_image_name }}
@@ -200,6 +215,10 @@ jobs:
             suite: cdc
             image_name: ${{ needs.params.outputs.test_image_name }}
             pg_version: ${{ needs.params.outputs.pg16_version }}
+          - make: installcheck
+            suite: cdc
+            image_name: ${{ needs.params.outputs.test_image_name }}
+            pg_version: ${{ needs.params.outputs.pg17_version }}
           - make: check-query-generator
             pg_version: ${{ needs.params.outputs.pg14_version }}
             suite: regress
@@ -212,6 +231,10 @@ jobs:
             pg_version: ${{ needs.params.outputs.pg16_version }}
             suite: regress
             image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: check-query-generator
+            pg_version: ${{ needs.params.outputs.pg17_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
     runs-on: ubuntu-20.04
     container:
       image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}"
@@ -255,6 +278,7 @@ jobs:
           - ${{ needs.params.outputs.pg14_version }}
           - ${{ needs.params.outputs.pg15_version }}
           - ${{ needs.params.outputs.pg16_version }}
+          - ${{ needs.params.outputs.pg17_version }}
         parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs
     steps:
     - uses: actions/[email protected]
@@ -303,6 +327,10 @@ jobs:
             new_pg_major: 16
           - old_pg_major: 14
             new_pg_major: 16
+          - old_pg_major: 16
+            new_pg_major: 17
+          - old_pg_major: 15
+            new_pg_major: 17
     env:
       old_pg_major: ${{ matrix.old_pg_major }}
       new_pg_major: ${{ matrix.new_pg_major }}
@@ -386,7 +414,7 @@ jobs:
       CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
     runs-on: ubuntu-20.04
     container:
-      image: ${{ needs.params.outputs.test_image_name }}:${{ fromJson(needs.params.outputs.pg16_version).full }}${{ needs.params.outputs.image_suffix }}
+      image: ${{ needs.params.outputs.test_image_name }}:${{ fromJson(needs.params.outputs.pg17_version).full }}${{ needs.params.outputs.image_suffix }}
     needs:
       - params
       - test-citus
@@ -481,7 +509,7 @@ jobs:
     name: Test flakyness
     runs-on: ubuntu-20.04
     container:
-      image: ${{ needs.params.outputs.fail_test_image_name }}:${{ needs.params.outputs.pg16_version }}${{ needs.params.outputs.image_suffix }}
+      image: ${{ needs.params.outputs.fail_test_image_name }}:${{ needs.params.outputs.pg17_version }}${{ needs.params.outputs.image_suffix }}
       options: --user root
     env:
       runs: 8

configure

@@ -2588,7 +2588,7 @@ fi
 if test "$with_pg_version_check" = no; then
     { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num (skipped compatibility check)" >&5
 $as_echo "$as_me: building against PostgreSQL $version_num (skipped compatibility check)" >&6;}
-elif test "$version_num" != '14' -a "$version_num" != '15' -a  "$version_num" != '16'; then
+elif test "$version_num" != '14' -a "$version_num" != '15' -a  "$version_num" != '16' -a  "$version_num" != '17'; then
    as_fn_error $? "Citus is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5
 else
    { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num" >&5

configure.ac

@@ -80,7 +80,7 @@ AC_SUBST(with_pg_version_check)
 
 if test "$with_pg_version_check" = no; then
     AC_MSG_NOTICE([building against PostgreSQL $version_num (skipped compatibility check)])
-elif test "$version_num" != '14' -a "$version_num" != '15' -a  "$version_num" != '16'; then
+elif test "$version_num" != '14' -a "$version_num" != '15' -a  "$version_num" != '16' -a  "$version_num" != '17'; then
    AC_MSG_ERROR([Citus is not compatible with the detected PostgreSQL version ${version_num}.])
 else
    AC_MSG_NOTICE([building against PostgreSQL $version_num])

src/backend/columnar/columnar_tableam.c

@@ -1437,7 +1437,19 @@ columnar_scan_analyze_next_block(TableScanDesc scan,
 	 * to pages boundaries. So not much to do here. We return true anyway
 	 * so acquire_sample_rows() in analyze.c would call our
 	 * columnar_scan_analyze_next_tuple() callback.
+	 * In PG17, we return false in case there is no buffer left, since
+	 * the outer loop changed in acquire_sample_rows(), and it is
+	 * expected for the scan_analyze_next_block function to check whether
+	 * there are any blocks left in the block sampler.
 	 */
+#if PG_VERSION_NUM >= PG_VERSION_17
+	Buffer buf = read_stream_next_buffer(stream, NULL);
+	if (!BufferIsValid(buf))
+	{
+		return false;
+	}
+	ReleaseBuffer(buf);
+#endif
 	return true;
 }
 
@@ -2244,7 +2256,9 @@ ColumnarProcessAlterTable(AlterTableStmt *alterTableStmt, List **columnarOptions
 									"Specify SET ACCESS METHOD before storage parameters, or use separate ALTER TABLE commands.")));
 			}
 
-			destIsColumnar = (strcmp(alterTableCmd->name, COLUMNAR_AM_NAME) == 0);
+			destIsColumnar = (strcmp(alterTableCmd->name ? alterTableCmd->name :
+									 default_table_access_method,
+									 COLUMNAR_AM_NAME) == 0);
 
 			if (srcIsColumnar && !destIsColumnar)
 			{

src/backend/distributed/commands/collation.c

@@ -132,6 +132,7 @@ CreateCollationDDLInternal(Oid collationId, Oid *collowner, char **quotedCollati
 	char *schemaName = get_namespace_name(collnamespace);
 	*quotedCollationName = quote_qualified_identifier(schemaName, collname);
 	const char *providerString =
+		collprovider == COLLPROVIDER_BUILTIN ? "builtin" :
 		collprovider == COLLPROVIDER_DEFAULT ? "default" :
 		collprovider == COLLPROVIDER_ICU ? "icu" :
 		collprovider == COLLPROVIDER_LIBC ? "libc" : NULL;

src/include/pg_version_compat.h

@@ -144,6 +144,8 @@ getStxstattarget_compat(HeapTuple tup)
 #define getProcNo_compat(a) (a->pgprocno)
 #define getLxid_compat(a) (a->lxid)
 
+#define COLLPROVIDER_BUILTIN 'b'
+
 #endif
 
 #if PG_VERSION_NUM >= PG_VERSION_16

src/test/regress/citus_tests/common.py

@@ -93,6 +93,7 @@ def get_pg_major_version():
     14: "10.2.0",
     15: "11.1.5",
     16: "12.1devel",
+    17: "12.1devel",
 }
 
 OLDEST_SUPPORTED_CITUS_VERSION = OLDEST_SUPPORTED_CITUS_VERSION_MATRIX[PG_MAJOR_VERSION]

src/test/regress/expected/dml_recursive.out

@@ -1,3 +1,9 @@
+-- Two alternative test outputs:
+-- dml_recursive_0.out for PG16 and before
+-- dml_recursive.out for PG17
+-- related commit
+-- PostgreSQL 17 includes an enhancement that allows the optimizer to transform correlated IN subqueries into more efficient join operations.
+-- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=9f1337639
 CREATE SCHEMA recursive_dml_queries;
 SET search_path TO recursive_dml_queries, public;
 SET citus.next_shard_id TO 2370000;
@@ -292,7 +298,29 @@ FROM
 	) as baz
 	) as foo WHERE second_distributed_table.tenant_id = foo.tenant_id
 RETURNING *;
-ERROR:  complex joins are only supported when all distributed tables are co-located and joined on their distribution columns
+ tenant_id | dept |          info          | tenant_id
+---------------------------------------------------------------------
+ 14        |    3 | {"f1": 14, "f2": 196}  | 14
+ 23        |    5 | {"f1": 23, "f2": 529}  | 23
+ 24        |    6 | {"f1": 24, "f2": 576}  | 24
+ 3         |    0 | {"f1": 3, "f2": 9}     | 3
+ 33        |    8 | {"f1": 33, "f2": 1089} | 33
+ 34        |    8 | {"f1": 34, "f2": 1156} | 34
+ 4         |    1 | {"f1": 4, "f2": 16}    | 4
+ 43        |   10 | {"f1": 43, "f2": 1849} | 43
+ 44        |   11 | {"f1": 44, "f2": 1936} | 44
+ 53        |   13 | {"f1": 53, "f2": 2809} | 53
+ 54        |   13 | {"f1": 54, "f2": 2916} | 54
+ 63        |   15 | {"f1": 63, "f2": 3969} | 63
+ 64        |   16 | {"f1": 64, "f2": 4096} | 64
+ 73        |   18 | {"f1": 73, "f2": 5329} | 73
+ 74        |   18 | {"f1": 74, "f2": 5476} | 74
+ 83        |   20 | {"f1": 83, "f2": 6889} | 83
+ 84        |   21 | {"f1": 84, "f2": 7056} | 84
+ 93        |   23 | {"f1": 93, "f2": 8649} | 93
+ 94        |   23 | {"f1": 94, "f2": 8836} | 94
+(19 rows)
+
 -- we don't support subqueries/CTEs inside VALUES
 INSERT INTO
 	second_distributed_table (tenant_id, dept)