diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 7fb2abebbbb2..4db0737c959d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -40,3 +40,33 @@ At a minimum, the following information should be added (but add more as needed) > Information about CI checks: https://clickhouse.com/docs/en/development/continuous-integration/ + +--- +### Modify your CI run: +##### NOTE: +- if your merge the PR with modified CI you **MUST** know what you are doing. +- modifiers can be applied only if set before CI starts +- remove `!` to apply +- return all `!` to restore defaults +``` +!#ci_set_ - to run only preconfigured set of tests, e.g.: +!#ci_set_arm - to run only integration tests on ARM +!#ci_set_integration - to run only integration tests on AMD +!#ci_set_analyzer - to run only tests for analyzer +NOTE: you can configure your own ci set +``` +``` +!#job_ - to run only specified job, e.g.: +!#job_stateless_tests_release +!#job_package_debug +!#job_style_check +!#job_integration_tests_asan +``` +``` +!#batch_2 - to run only 2nd batch for all multi-batch jobs +!#btach_1_2_3 - to run only 1, 2, 3rd batch for all multi-batch jobs +``` +``` +!#no_merge_commit - to disable merge commit (no merge from master) +!#do_not_test - to disable whole CI (except style check) +``` diff --git a/.gitmessage b/.gitmessage index 2ad30596de69..797446edd49e 100644 --- a/.gitmessage +++ b/.gitmessage @@ -26,4 +26,4 @@ ## To run only specified batches for multi-batch job(s) #batch_2 -#btach_1_2_3 +#batch_1_2_3 diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 29906e6571f8..cd63514cb6af 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1407,15 +1407,25 @@ def _concurrent_create_status(job: str, batch: int, num_batches: int) -> None: print("... CI report update - done") -def _fetch_commit_tokens(message: str) -> List[str]: - pattern = r"#[\w-]+" - matches = [match[1:] for match in re.findall(pattern, message)] +def _fetch_commit_tokens(message: str, pr_info: PRInfo) -> List[str]: + pattern = r"([^!]|^)#(\w+)" + matches = [match[-1] for match in re.findall(pattern, message)] res = [ match for match in matches if match in Labels or match.startswith("job_") or match.startswith("batch_") ] - return res + print(f"CI modifyers from commit message: [{res}]") + res_2 = [] + if pr_info.is_pr(): + matches = [match[-1] for match in re.findall(pattern, pr_info.body)] + res_2 = [ + match + for match in matches + if match in Labels or match.startswith("job_") or match.startswith("batch_") + ] + print(f"CI modifyers from PR body: [{res_2}]") + return list(set(res + res_2)) def _upload_build_artifacts( @@ -1701,8 +1711,7 @@ def main() -> int: message = args.commit_message or git_runner.run( f"{GIT_PREFIX} log {pr_info.sha} --format=%B -n 1" ) - tokens = _fetch_commit_tokens(message) - print(f"Commit message tokens: [{tokens}]") + tokens = _fetch_commit_tokens(message, pr_info) if Labels.NO_MERGE_COMMIT in tokens and CI: git_runner.run(f"{GIT_PREFIX} checkout {pr_info.sha}") git_ref = git_runner.run(f"{GIT_PREFIX} rev-parse HEAD") diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 9bd30f3c58e9..84f2db4002dc 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -316,6 +316,9 @@ def is_release(self) -> bool: def is_release_branch(self) -> bool: return self.number == 0 + def is_pr(self): + return self.event_type == EventType.PULL_REQUEST + def is_scheduled(self): return self.event_type == EventType.SCHEDULE diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index d7f6fa998e98..05788aad5ea2 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -8,9 +8,8 @@ import sys from concurrent.futures import ProcessPoolExecutor from pathlib import Path -from typing import List, Tuple, Union +from typing import List, Tuple -import magic from docker_images_helper import get_docker_image, pull_image from env_helper import CI, REPO_COPY, TEMP_PATH from git_helper import GIT_PREFIX, git_runner @@ -96,32 +95,34 @@ def commit_push_staged(pr_info: PRInfo) -> None: git_runner(push_cmd) -def is_python(file: Union[Path, str]) -> bool: +def is_python(file: str) -> bool: """returns if the changed file in the repository is python script""" # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 - try: - return bool( - magic.from_file(os.path.join(REPO_COPY, file), mime=True) - == "text/x-script.python" - ) - except IsADirectoryError: - # Process submodules w/o errors - return False - - -def is_shell(file: Union[Path, str]) -> bool: + # try: + # return bool( + # magic.from_file(os.path.join(REPO_COPY, file), mime=True) + # == "text/x-script.python" + # ) + # except IsADirectoryError: + # # Process submodules w/o errors + # return False + return file.endswith(".py") + + +def is_shell(file: str) -> bool: """returns if the changed file in the repository is shell script""" # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 - try: - return bool( - magic.from_file(os.path.join(REPO_COPY, file), mime=True) - == "text/x-shellscript" - ) - except IsADirectoryError: - # Process submodules w/o errors - return False + # try: + # return bool( + # magic.from_file(os.path.join(REPO_COPY, file), mime=True) + # == "text/x-shellscript" + # ) + # except IsADirectoryError: + # # Process submodules w/o errors + # return False + return file.endswith(".sh") def main():