Merge branch 'master' into modify-engine

.github/workflows/backport_branches.yml

@@ -11,7 +11,7 @@ on: # yamllint disable-line rule:truthy
       - 'backport/**'
 jobs:
   RunConfig:
-    runs-on: [self-hosted, style-checker]
+    runs-on: [self-hosted, style-checker-aarch64]
     outputs:
       data: ${{ steps.runconfig.outputs.CI_DATA }}
     steps:

.github/workflows/master.yml

@@ -11,7 +11,7 @@ on: # yamllint disable-line rule:truthy
       - 'master'
 jobs:
   RunConfig:
-    runs-on: [self-hosted, style-checker]
+    runs-on: [self-hosted, style-checker-aarch64]
     outputs:
       data: ${{ steps.runconfig.outputs.CI_DATA }}
     steps:
@@ -327,6 +327,7 @@ jobs:
       run_command: |
         python3 build_report_check.py "$CHECK_NAME"
   MarkReleaseReady:
+    if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
     needs:
       - BuilderBinDarwin
       - BuilderBinDarwinAarch64
@@ -374,14 +375,6 @@ jobs:
       test_name: Stateless tests (release)
       runner_type: func-tester
       data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatelessTestCoverage:
-    needs: [RunConfig, BuilderDebReleaseCoverage]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateless tests (coverage)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
   FunctionalStatelessTestReleaseDatabaseReplicated:
     needs: [RunConfig, BuilderDebRelease]
     if: ${{ !failure() && !cancelled() }}
@@ -482,14 +475,6 @@ jobs:
       test_name: Stateful tests (release)
       runner_type: func-tester
       data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatefulTestCoverage:
-    needs: [RunConfig, BuilderDebReleaseCoverage]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateful tests (coverage)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
   FunctionalStatefulTestAarch64:
     needs: [RunConfig, BuilderDebAarch64]
     if: ${{ !failure() && !cancelled() }}

.github/workflows/nightly.yml

@@ -14,7 +14,7 @@ jobs:
     # The task for having a preserved ENV and event.json for later investigation
     uses: ./.github/workflows/debug.yml
   RunConfig:
-    runs-on: [self-hosted, style-checker]
+    runs-on: [self-hosted, style-checker-aarch64]
     outputs:
       data: ${{ steps.runconfig.outputs.CI_DATA }}
     steps:

.github/workflows/pull_request.yml

@@ -18,7 +18,7 @@ on:  # yamllint disable-line rule:truthy
 ##########################################################################################
 jobs:
   RunConfig:
-    runs-on: [self-hosted, style-checker]
+    runs-on: [self-hosted, style-checker-aarch64]
     outputs:
       data: ${{ steps.runconfig.outputs.CI_DATA }}
     steps:
@@ -391,14 +391,6 @@ jobs:
       test_name: Stateless tests (release)
       runner_type: func-tester
       data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatelessTestCoverage:
-    needs: [RunConfig, BuilderDebReleaseCoverage]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateless tests (coverage)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
   FunctionalStatelessTestReleaseDatabaseReplicated:
     needs: [RunConfig, BuilderDebRelease]
     if: ${{ !failure() && !cancelled() }}
@@ -500,21 +492,9 @@ jobs:
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/reusable_test.yml
     with:
-      test_name: tests bugfix validate check
+      test_name: Bugfix validation
       runner_type: func-tester
       data: ${{ needs.RunConfig.outputs.data }}
-      additional_envs: |
-        KILL_TIMEOUT=3600
-      run_command: |
-        TEMP_PATH="${TEMP_PATH}/integration" \
-          python3 integration_test_check.py "Integration $CHECK_NAME" \
-            --validate-bugfix --post-commit-status=file || echo "ignore exit code"
-
-        TEMP_PATH="${TEMP_PATH}/stateless" \
-          python3 functional_test_check.py "Stateless $CHECK_NAME" "$KILL_TIMEOUT" \
-            --validate-bugfix --post-commit-status=file || echo "ignore exit code"
-
-        python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/functional_commit_status.tsv" "${TEMP_PATH}/integration/integration_commit_status.tsv"
 ##############################################################################################
 ############################ FUNCTIONAl STATEFUL TESTS #######################################
 ##############################################################################################
@@ -526,14 +506,6 @@ jobs:
       test_name: Stateful tests (release)
       runner_type: func-tester
       data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatefulTestCoverage:
-    needs: [RunConfig, BuilderDebReleaseCoverage]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateful tests (coverage)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
   FunctionalStatefulTestAarch64:
     needs: [RunConfig, BuilderDebAarch64]
     if: ${{ !failure() && !cancelled() }}
@@ -915,6 +887,7 @@ jobs:
       - BuilderSpecialReport
       - DocsCheck
       - FastTest
+      - TestsBugfixCheck
       - FunctionalStatelessTestDebug
       - FunctionalStatelessTestRelease
       - FunctionalStatelessTestReleaseDatabaseReplicated

.github/workflows/release_branches.yml

@@ -14,7 +14,7 @@ on: # yamllint disable-line rule:truthy
 
 jobs:
   RunConfig:
-    runs-on: [self-hosted, style-checker]
+    runs-on: [self-hosted, style-checker-aarch64]
     outputs:
       data: ${{ steps.runconfig.outputs.CI_DATA }}
     steps:
@@ -228,6 +228,7 @@ jobs:
       run_command: |
         python3 build_report_check.py "$CHECK_NAME"
   MarkReleaseReady:
+    if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
     needs:
       - BuilderBinDarwin
       - BuilderBinDarwinAarch64

.gitmessage

@@ -1,20 +1,28 @@
 
 
-### CI modificators (add a leading space to apply):
+### CI modificators (add a leading space to apply) ###
 
 ## To avoid a merge commit in CI:
 #no_merge_commit
 
 ## To discard CI cache:
 #no_ci_cache
 
+## To not test (only style check):
+#do_not_test
+
 ## To run specified set of tests in CI:
 #ci_set_<SET_NAME>
 #ci_set_reduced
 #ci_set_arm
+#ci_set_integration
 
 ## To run specified job in CI:
 #job_<JOB NAME>
 #job_stateless_tests_release
 #job_package_debug
 #job_integration_tests_asan
+
+## To run only specified batches for multi-batch job(s)
+#batch_2
+#btach_1_2_3

base/base/Decimal_fwd.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <base/types.h>
+#include <base/extended_types.h>
 
 namespace wide
 {
@@ -44,3 +45,8 @@ concept is_over_big_int =
     || std::is_same_v<T, Decimal128>
     || std::is_same_v<T, Decimal256>;
 }
+
+template <> struct is_signed<DB::Decimal32> { static constexpr bool value = true; };
+template <> struct is_signed<DB::Decimal64> { static constexpr bool value = true; };
+template <> struct is_signed<DB::Decimal128> { static constexpr bool value = true; };
+template <> struct is_signed<DB::Decimal256> { static constexpr bool value = true; };

contrib/libssh-cmake/CMakeLists.txt

@@ -8,23 +8,11 @@ endif()
 set(LIB_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libssh")
 set(LIB_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/libssh")
 
-project(libssh VERSION 0.9.7 LANGUAGES C)
-
-# global needed variable
-set(APPLICATION_NAME ${PROJECT_NAME})
-
-# SOVERSION scheme: CURRENT.AGE.REVISION
-#   If there was an incompatible interface change:
-#     Increment CURRENT. Set AGE and REVISION to 0
-#   If there was a compatible interface change:
-#     Increment AGE. Set REVISION to 0
-#   If the source code was changed, but there were no interface changes:
-#     Increment REVISION.
-set(LIBRARY_VERSION "4.8.7")
-set(LIBRARY_SOVERSION "4")
+# Set CMake variables which are used in libssh_version.h.cmake
+project(libssh VERSION 0.9.8 LANGUAGES C)
 
-# Copy library files to a lib sub-directory
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${LIB_BINARY_DIR}/lib")
+set(LIBRARY_VERSION "4.8.8")
+set(LIBRARY_SOVERSION "4")
 
 set(CMAKE_THREAD_PREFER_PTHREADS ON)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
@@ -33,7 +21,87 @@ set(WITH_ZLIB OFF)
 set(WITH_SYMBOL_VERSIONING OFF)
 set(WITH_SERVER ON)
 
-include(IncludeSources.cmake)
+set(libssh_SRCS
+    ${LIB_SOURCE_DIR}/src/agent.c
+    ${LIB_SOURCE_DIR}/src/auth.c
+    ${LIB_SOURCE_DIR}/src/base64.c
+    ${LIB_SOURCE_DIR}/src/bignum.c
+    ${LIB_SOURCE_DIR}/src/buffer.c
+    ${LIB_SOURCE_DIR}/src/callbacks.c
+    ${LIB_SOURCE_DIR}/src/channels.c
+    ${LIB_SOURCE_DIR}/src/client.c
+    ${LIB_SOURCE_DIR}/src/config.c
+    ${LIB_SOURCE_DIR}/src/connect.c
+    ${LIB_SOURCE_DIR}/src/connector.c
+    ${LIB_SOURCE_DIR}/src/curve25519.c
+    ${LIB_SOURCE_DIR}/src/dh.c
+    ${LIB_SOURCE_DIR}/src/ecdh.c
+    ${LIB_SOURCE_DIR}/src/error.c
+    ${LIB_SOURCE_DIR}/src/getpass.c
+    ${LIB_SOURCE_DIR}/src/init.c
+    ${LIB_SOURCE_DIR}/src/kdf.c
+    ${LIB_SOURCE_DIR}/src/kex.c
+    ${LIB_SOURCE_DIR}/src/known_hosts.c
+    ${LIB_SOURCE_DIR}/src/knownhosts.c
+    ${LIB_SOURCE_DIR}/src/legacy.c
+    ${LIB_SOURCE_DIR}/src/log.c
+    ${LIB_SOURCE_DIR}/src/match.c
+    ${LIB_SOURCE_DIR}/src/messages.c
+    ${LIB_SOURCE_DIR}/src/misc.c
+    ${LIB_SOURCE_DIR}/src/options.c
+    ${LIB_SOURCE_DIR}/src/packet.c
+    ${LIB_SOURCE_DIR}/src/packet_cb.c
+    ${LIB_SOURCE_DIR}/src/packet_crypt.c
+    ${LIB_SOURCE_DIR}/src/pcap.c
+    ${LIB_SOURCE_DIR}/src/pki.c
+    ${LIB_SOURCE_DIR}/src/pki_container_openssh.c
+    ${LIB_SOURCE_DIR}/src/poll.c
+    ${LIB_SOURCE_DIR}/src/session.c
+    ${LIB_SOURCE_DIR}/src/scp.c
+    ${LIB_SOURCE_DIR}/src/socket.c
+    ${LIB_SOURCE_DIR}/src/string.c
+    ${LIB_SOURCE_DIR}/src/threads.c
+    ${LIB_SOURCE_DIR}/src/wrapper.c
+    ${LIB_SOURCE_DIR}/src/external/bcrypt_pbkdf.c
+    ${LIB_SOURCE_DIR}/src/external/blowfish.c
+    ${LIB_SOURCE_DIR}/src/external/chacha.c
+    ${LIB_SOURCE_DIR}/src/external/poly1305.c
+    ${LIB_SOURCE_DIR}/src/chachapoly.c
+    ${LIB_SOURCE_DIR}/src/config_parser.c
+    ${LIB_SOURCE_DIR}/src/token.c
+    ${LIB_SOURCE_DIR}/src/pki_ed25519_common.c
+
+    ${LIB_SOURCE_DIR}/src/threads/noop.c
+    ${LIB_SOURCE_DIR}/src/threads/pthread.c
+
+    # LIBCRYPT specific
+    ${libssh_SRCS}
+    ${LIB_SOURCE_DIR}/src/threads/libcrypto.c
+    ${LIB_SOURCE_DIR}/src/pki_crypto.c
+    ${LIB_SOURCE_DIR}/src/ecdh_crypto.c
+    ${LIB_SOURCE_DIR}/src/libcrypto.c
+    ${LIB_SOURCE_DIR}/src/dh_crypto.c
+
+    ${LIB_SOURCE_DIR}/src/options.c
+    ${LIB_SOURCE_DIR}/src/server.c
+    ${LIB_SOURCE_DIR}/src/bind.c
+    ${LIB_SOURCE_DIR}/src/bind_config.c
+)
+
+if (NOT (ENABLE_OPENSSL OR ENABLE_OPENSSL_DYNAMIC))
+    add_compile_definitions(USE_BORINGSSL=1)
+endif()
+
+configure_file(${LIB_SOURCE_DIR}/include/libssh/libssh_version.h.cmake ${LIB_BINARY_DIR}/include/libssh/libssh_version.h @ONLY)
+
+add_library(_ssh STATIC ${libssh_SRCS})
+add_library(ch_contrib::ssh ALIAS _ssh)
+
+target_link_libraries(_ssh PRIVATE OpenSSL::Crypto)
+
+target_include_directories(_ssh PUBLIC "${LIB_SOURCE_DIR}/include" "${LIB_BINARY_DIR}/include")
+
+# These headers need to be generated using the native build system on each platform.
 if (OS_LINUX)
     if (ARCH_AMD64)
         if (USE_MUSL)
@@ -63,7 +131,3 @@ elseif (OS_FREEBSD)
 else ()
     message(FATAL_ERROR "Platform is not supported")
 endif()
-
-configure_file(${LIB_SOURCE_DIR}/include/libssh/libssh_version.h.cmake
-               ${LIB_BINARY_DIR}/include/libssh/libssh_version.h
-               @ONLY)