diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c44ff271..f5cfe950 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -47,7 +47,7 @@ jobs: github.repository == 'MariaDB/buildbot' run: | rsync -a --progress --delete --exclude-from=rsync.exclude -e "ssh -p ${{ secrets.SERVER_PORT }}" ./ ${{ secrets.USER }}@${{ secrets.SERVER_IP }}:/srv/dev/ - ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose/ && ./generate-config.py" + ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose/ && ./generate-config.py --env=dev" ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/ && ./define_masters.py" ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose/ && ln -sf .. buildbot" - name: start stack @@ -55,7 +55,7 @@ jobs: github.ref == 'refs/heads/dev' && github.repository == 'MariaDB/buildbot' run: | - ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose && docker-compose pull && docker-compose up -d" + ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose && docker-compose pull && docker-compose --env-file .env.dev up -d" - name: clean if: > always() && diff --git a/.gitignore b/.gitignore index 9e86ac1d..c35c6698 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ venv workers autogen master-config.yaml +.venv diff --git a/Dockerfile b/Dockerfile index 3a66575a..cae924b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ && if [ "$master_type" = "master-web" ]; then \ make frontend; \ fi \ - && pip install --no-cache-dir wheel \ + && pip install --no-cache-dir python-dotenv wheel \ && pip install --no-cache-dir autobahn==20.7.1 PyJWT==1.7.1 \ && cd master && python setup.py bdist_wheel \ && pip install --no-cache-dir dist/*.whl \ diff --git a/constants.py b/constants.py index c5c7b9c2..a9948507 100644 --- a/constants.py +++ b/constants.py @@ -170,15 +170,15 @@ builders_upgrade = [] builders_autobake = [] all_platforms = set() -for os in os_info: - for arch in os_info[os]['arch']: +for os_i in os_info: + for arch in os_info[os_i]['arch']: all_platforms.add(arch) - builder_name_autobake = arch + '-' + os + '-' + os_info[os]['type'] + '-autobake' + builder_name_autobake = arch + '-' + os_i + '-' + os_info[os_i]['type'] + '-autobake' builders_autobake.append(builder_name_autobake) # Currently there are no VMs for x86 and s390x and OpenSUSE and SLES addInstall = True - if 'has_install' in os_info[os]: - addInstall = os_info[os]['has_install'] + if 'has_install' in os_info[os_i]: + addInstall = os_info[os_i]['has_install'] if arch not in ['s390x', 'x86'] and addInstall: builders_install.append(builder_name_autobake + '-install') builders_upgrade.append(builder_name_autobake + '-minor-upgrade') diff --git a/docker-compose/.env b/docker-compose/.env new file mode 100644 index 00000000..dbbad0a7 --- /dev/null +++ b/docker-compose/.env @@ -0,0 +1,4 @@ +TITLE="MariaDB CI" +TITLE_URL=https://github.com/MariaDB/server +BUILDMASTER_URL=https://buildbot.mariadb.org/ +MQ_ROUTER_URL=ws://localhost:8085/ws \ No newline at end of file diff --git a/docker-compose/.env.dev b/docker-compose/.env.dev new file mode 100644 index 00000000..be21d369 --- /dev/null +++ b/docker-compose/.env.dev @@ -0,0 +1,4 @@ +TITLE="MariaDB CI" +TITLE_URL=https://github.com/MariaDB/server +BUILDMASTER_URL=https://buildbot.dev.mariadb.org/ +MQ_ROUTER_URL=ws://crossbar:8085/ws \ No newline at end of file diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index 77c7634b..7d0093df 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -31,6 +31,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master-web restart: unless-stopped container_name: master-web + environment: + - MQ_ROUTER_URL + - TITLE_URL + - TITLE + - BUILDMASTER_URL volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -49,6 +54,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: autogen_aarch64-master-0 + environment: + - MQ_ROUTER_URL + - TITLE_URL + - TITLE + - BUILDMASTER_URL volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -69,6 +79,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: autogen_amd64-master-0 + environment: + - MQ_ROUTER_URL + - TITLE_URL + - TITLE + - BUILDMASTER_URL volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -89,6 +104,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: autogen_amd64-master-1 + environment: + - MQ_ROUTER_URL + - TITLE_URL + - TITLE + - BUILDMASTER_URL volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -109,6 +129,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: autogen_ppc64le-master-0 + environment: + - MQ_ROUTER_URL + - TITLE_URL + - TITLE + - BUILDMASTER_URL volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -129,6 +154,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: autogen_s390x-master-0 + environment: + - MQ_ROUTER_URL + - TITLE_URL + - TITLE + - BUILDMASTER_URL volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -149,6 +179,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: autogen_x86-master-0 + environment: + - MQ_ROUTER_URL + - TITLE_URL + - TITLE + - BUILDMASTER_URL volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -169,6 +204,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: master-docker-nonstandard + environment: + - MQ_ROUTER_URL + - TITLE_URL + - TITLE + - BUILDMASTER_URL volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -189,6 +229,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: master-galera + environment: + - MQ_ROUTER_URL + - TITLE_URL + - TITLE + - BUILDMASTER_URL volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master @@ -209,6 +254,11 @@ services: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped container_name: master-protected-branches + environment: + - MQ_ROUTER_URL + - TITLE_URL + - TITLE + - BUILDMASTER_URL volumes: - ./logs:/var/log/buildbot - ./buildbot/:/srv/buildbot/master diff --git a/docker-compose/generate-config.py b/docker-compose/generate-config.py index f77f21ed..2b09f88b 100755 --- a/docker-compose/generate-config.py +++ b/docker-compose/generate-config.py @@ -1,6 +1,10 @@ #!/usr/bin/env python3 +import argparse +import os -master_directories = [ +from dotenv import load_dotenv + +MASTER_DIRECTORIES = [ "autogen/aarch64-master-0", "autogen/amd64-master-0", "autogen/amd64-master-1", @@ -15,7 +19,7 @@ "master-protected-branches", ] -start_template = """ +START_TEMPLATE = """ --- version: "3.7" services: @@ -63,7 +67,7 @@ - crossbar """ -docker_compose_template = """ +DOCKER_COMPOSE_TEMPLATE = """ {master_name}: image: quay.io/mariadb-foundation/bb-master:master restart: unless-stopped @@ -85,7 +89,7 @@ - crossbar """ -end_template = """ +END_TEMPLATE = """ networks: net_front: driver: bridge @@ -108,21 +112,56 @@ com.docker.network.bridge.name: "br_bb_back" """ -# Generate startup scripts and Docker Compose pieces for each master directory -with open("docker-compose.yaml", "w") as f: - f.write("# This is an autogenerated file. Do not edit it manually. Use `python generate-config.py` instead.") - f.write(start_template) - port = 8011 - for master_directory in master_directories: - master_name = master_directory.replace("/", "_") - - # Generate Docker Compose piece - docker_compose_piece = docker_compose_template.format( - master_name=master_name, - master_directory=master_directory, - port=port, - ) - port += 1 - - f.write(docker_compose_piece) - f.write(end_template) +# Function to construct environment section for Docker Compose +def construct_env_section(env_vars): + env_section = " environment:\n" + for key, value in env_vars.items(): + env_section += f" - {key}\n" + return env_section.rstrip('\n') + +def main(args): + # Capture the current environment variables' keys + current_env_keys = set(os.environ.keys()) + + # Load environment variables from the corresponding .env file + env_file = ".env" if args.env == "prod" else ".env.dev" + load_dotenv(env_file) + + # Determine the keys that were added by the .env file + new_keys = set(os.environ.keys()) - current_env_keys + + # Extract only the variables from the .env file + env_vars = {key: os.getenv(key) for key in new_keys} + + # Modify the start_template to include the environment variables for master-web + master_env_section = construct_env_section(env_vars) + start_template = START_TEMPLATE.replace("container_name: master-web", f"container_name: master-web\n{master_env_section}") + + # Modify the docker_compose_template to include the environment variables + docker_compose_template = DOCKER_COMPOSE_TEMPLATE.replace("container_name: {master_name}", f"container_name: {{master_name}}\n{master_env_section}") + + # Generate startup scripts and Docker Compose pieces for each master directory + with open("docker-compose.yaml", "w") as f: + f.write("# This is an autogenerated file. Do not edit it manually. Use `python generate-config.py` instead.") + f.write(start_template) + port = 8011 + for master_directory in MASTER_DIRECTORIES: + master_name = master_directory.replace("/", "_") + + # Generate Docker Compose piece + docker_compose_piece = docker_compose_template.format( + master_name=master_name, + master_directory=master_directory, + port=port, + ) + port += 1 + + f.write(docker_compose_piece) + f.write(END_TEMPLATE) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate Docker Compose configuration.") + parser.add_argument("--env", choices=["prod", "dev"], default="dev", help="Choose the environment (prod/dev). Default is dev.") + + args = parser.parse_args() + main(args) diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg index 2f3bd721..4edf7207 100644 --- a/master-docker-nonstandard/master.cfg +++ b/master-docker-nonstandard/master.cfg @@ -7,10 +7,12 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub from buildbot.process.remotecommand import RemoteCommand +from datetime import timedelta from twisted.internet import defer -import sys + import docker -from datetime import timedelta +import os +import sys sys.path.insert(0, '/srv/buildbot/master') sys.setrecursionlimit(10000) @@ -51,15 +53,14 @@ c['services'].append(gs) # the 'title' string will appear at the top of this buildbot installation's # home pages (linked to the 'titleURL'). -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" +c['title'] = os.getenv('TITLE', default="MariaDB CI") +c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server is visible. This typically uses the port number set in # the 'www' entry below, but with an externally-visible host name which the # buildbot cannot figure out without some help. - -c['buildbotURL'] = "https://buildbot.mariadb.org/" +c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") # 'protocols' contains information about protocols which master will use for # communicating with workers. You must define at least 'port' option that workers @@ -758,7 +759,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://localhost:8085/ws', + 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/master-galera/dockerfiles b/master-galera/dockerfiles deleted file mode 120000 index f8f80794..00000000 --- a/master-galera/dockerfiles +++ /dev/null @@ -1 +0,0 @@ -../dockerfiles \ No newline at end of file diff --git a/master-galera/master.cfg b/master-galera/master.cfg index 8867b5b2..66672dcd 100644 --- a/master-galera/master.cfg +++ b/master-galera/master.cfg @@ -7,11 +7,12 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub from buildbot.process.remotecommand import RemoteCommand +from datetime import timedelta from twisted.internet import defer -import sys -import os + import docker -from datetime import timedelta +import os +import sys sys.setrecursionlimit(10000) @@ -32,18 +33,14 @@ exec(open("../master-private.cfg").read(), config, { }) # the 'title' string will appear at the top of this buildbot installation's # home pages (linked to the 'titleURL'). -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" +c['title'] = os.getenv('TITLE', default="MariaDB CI") +c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server is visible. This typically uses the port number set in # the 'www' entry below, but with an externally-visible host name which the # buildbot cannot figure out without some help. - -c['buildbotURL'] = "https://buildbot.dev.mariadb.org/" - -# Custom plugin -# exec(open("grid.py").read()) +c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") # 'protocols' contains information about protocols which master will use for # communicating with workers. You must define at least 'port' option that workers @@ -142,10 +139,10 @@ for platform in all_platforms: for w_name in machines: for i in worker_ids: - for os in os_info: - if platform in os_info[os]['arch']: - quay_name = 'quay.io/mariadb-foundation/bb-worker:' + ''.join(os.split('-')) - os_name = os + for os_i in os_info: + if platform in os_info[os_i]['arch']: + quay_name = 'quay.io/mariadb-foundation/bb-worker:' + ''.join(os_i.split('-')) + os_name = os_i if "ubuntu" in quay_name: quay_name = quay_name[:-2] + '.' + quay_name[-2:] if platform == 'x86': @@ -214,19 +211,19 @@ f_rpm_build.addStep(steps.ShellCommand(name="cleanup", command="rm -r * .* 2> /d ####### BUILDERS LIST c['builders'] = [] -for os in os_info: - for arch in os_info[os]['arch']: - builder_name = 'gal-' + arch + '-' + os +for os_i in os_info: + for arch in os_info[os_i]['arch']: + builder_name = 'gal-' + arch + '-' + os_i assert builder_name in builders_galera - worker_name = arch + '-bbw-docker-' + os + worker_name = arch + '-bbw-docker-' + os_i if arch == 'amd64': - worker_name = 'x64-bbw-docker-' + os + worker_name = 'x64-bbw-docker-' + os_i if arch == 'x86': - worker_name = 'x64-bbw-docker-' + os + '-i386' + worker_name = 'x64-bbw-docker-' + os_i + '-i386' - if os_info[os]['type'] == 'rpm': + if os_info[os_i]['type'] == 'rpm': factory = f_rpm_build else: factory = f_deb_build @@ -234,7 +231,7 @@ for os in os_info: c['builders'].append( util.BuilderConfig(name=builder_name, workernames=workers[worker_name], - tags=[os, "galera", "gcc"], + tags=[os_i, "galera", "gcc"], collapseRequests=True, nextBuild=nextBuild, factory=factory)) @@ -245,7 +242,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://crossbar:8085/ws', + 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index 7279414f..b47ffb4c 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -7,10 +7,12 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub from buildbot.process.remotecommand import RemoteCommand +from datetime import timedelta from twisted.internet import defer -import sys + import docker -from datetime import timedelta +import os +import sys sys.setrecursionlimit(10000) @@ -31,15 +33,14 @@ exec(open("../master-private.cfg").read(), config, { }) # the 'title' string will appear at the top of this buildbot installation's # home pages (linked to the 'titleURL'). -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" +c['title'] = os.getenv('TITLE', default="MariaDB CI") +c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server is visible. This typically uses the port number set in # the 'www' entry below, but with an externally-visible host name which the # buildbot cannot figure out without some help. - -c['buildbotURL'] = "https://buildbot.dev.mariadb.org/" +c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") # 'protocols' contains information about protocols which master will use for # communicating with workers. You must define at least 'port' option that workers @@ -287,7 +288,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://crossbar:8085/ws', + 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/master-nonlatent/master.cfg b/master-nonlatent/master.cfg index dd2c82c9..d9816dec 100644 --- a/master-nonlatent/master.cfg +++ b/master-nonlatent/master.cfg @@ -7,10 +7,12 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub from buildbot.process.remotecommand import RemoteCommand +from datetime import timedelta from twisted.internet import defer -import sys + import docker -from datetime import timedelta +import os +import sys sys.setrecursionlimit(10000) @@ -36,18 +38,14 @@ exec(open("../master-private.cfg").read(), config, { }) # the 'title' string will appear at the top of this buildbot installation's # home pages (linked to the 'titleURL'). -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" +c['title'] = os.getenv('TITLE', default="MariaDB CI") +c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server is visible. This typically uses the port number set in # the 'www' entry below, but with an externally-visible host name which the # buildbot cannot figure out without some help. - -c['buildbotURL'] = "https://buildbot.mariadb.org/" - -# Custom plugin -# exec(open("grid.py").read()) +c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") # 'protocols' contains information about protocols which master will use for # communicating with workers. You must define at least 'port' option that workers @@ -344,7 +342,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://crossbar:8085/ws', + 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/master-protected-branches/master.cfg b/master-protected-branches/master.cfg index 26934121..ec6cda3b 100644 --- a/master-protected-branches/master.cfg +++ b/master-protected-branches/master.cfg @@ -7,10 +7,12 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub from buildbot.process.remotecommand import RemoteCommand +from datetime import timedelta from twisted.internet import defer -import sys + import docker -from datetime import timedelta +import os +import sys sys.setrecursionlimit(10000) @@ -52,18 +54,14 @@ c['services'].append(gs) # the 'title' string will appear at the top of this buildbot installation's # home pages (linked to the 'titleURL'). -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" +c['title'] = os.getenv('TITLE', default="MariaDB CI") +c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server is visible. This typically uses the port number set in # the 'www' entry below, but with an externally-visible host name which the # buildbot cannot figure out without some help. - -c['buildbotURL'] = "https://buildbot.mariadb.org/" - -# Custom plugin -# exec(open("grid.py").read()) +c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") # 'protocols' contains information about protocols which master will use for # communicating with workers. You must define at least 'port' option that workers @@ -339,7 +337,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://localhost:8085/ws', + 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/master-web/master.cfg b/master-web/master.cfg index bcc8ff6e..8d173c59 100644 --- a/master-web/master.cfg +++ b/master-web/master.cfg @@ -7,11 +7,12 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub from buildbot.process.remotecommand import RemoteCommand +from datetime import timedelta from twisted.internet import defer + +import docker import os import sys -import docker -from datetime import timedelta sys.setrecursionlimit(10000) @@ -27,9 +28,9 @@ exec(open("../master-private.cfg").read(), config, { }) ####### PROJECT IDENTITY -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" -c['buildbotURL'] = "https://buildbot.dev.mariadb.org/" +c['title'] = os.getenv('TITLE', default="MariaDB CI") +c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") +c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") # minimalistic config to activate web UI c['www'] = dict(port=8010, plugins=dict(waterfall_view={}, console_view={}, grid_view={}), custom_templates_dir='templates') @@ -78,7 +79,7 @@ c['multiMaster'] = True # Need to enable multimaster aware mq. Wamp is the only option for now. c['mq'] = { 'type' : 'wamp', - 'router_url': 'ws://crossbar:8080/ws', + 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/master.cfg b/master.cfg index 75895872..3b122891 100644 --- a/master.cfg +++ b/master.cfg @@ -7,10 +7,12 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub from buildbot.process.remotecommand import RemoteCommand +from datetime import timedelta from twisted.internet import defer -import sys + import docker -from datetime import timedelta +import os +import sys sys.setrecursionlimit(10000) @@ -55,18 +57,14 @@ c['services'].append(gs) # the 'title' string will appear at the top of this buildbot installation's # home pages (linked to the 'titleURL'). -c['title'] = "MariaDB CI" -c['titleURL'] = "https://github.com/MariaDB/server" +c['title'] = os.getenv('TITLE', default="MariaDB CI") +c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server is visible. This typically uses the port number set in # the 'www' entry below, but with an externally-visible host name which the # buildbot cannot figure out without some help. - -c['buildbotURL'] = "https://buildbot.mariadb.org/" - -# Custom plugin -# exec(open("grid.py").read()) +c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") # 'protocols' contains information about protocols which master will use for # communicating with workers. You must define at least 'port' option that workers @@ -262,7 +260,7 @@ c['multiMaster'] = True c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. 'type' : 'wamp', - 'router_url': 'ws://crossbar:8085/ws', + 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), 'realm': 'realm1', # valid are: none, critical, error, warn, info, debug, trace 'wamp_debug_level' : 'info' diff --git a/rsync.exclude b/rsync.exclude index 09ad3956..b43370c0 100644 --- a/rsync.exclude +++ b/rsync.exclude @@ -5,4 +5,4 @@ autogen ci_build_images docker-compose/logs docker-compose/mariadb -master-private.cfg +master-private.cfg \ No newline at end of file