Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: Refactor master cfg #673

Closed
wants to merge 16 commits into from
Closed
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Refactor createWorker from utils.py
Simplify and ensure it returns a 3-value tuple.
cvicentiu committed Jan 12, 2025
commit f69ee56af29d344299a0ed275cd45ca99db9ed88
22 changes: 10 additions & 12 deletions master-docker-nonstandard-2/master.cfg
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@ from twisted.internet import defer
import docker
import os
import sys
from collections import defaultdict

sys.path.insert(0, "/srv/buildbot/master")
sys.setrecursionlimit(10000)
@@ -121,7 +122,7 @@ c["schedulers"] = SCHEDULERS
# worker name and password must be configured on the worker.
c["workers"] = []

workers = {}
workers = defaultdict(list)


def addWorker(
@@ -133,7 +134,7 @@ def addWorker(
save_packages=False,
shm_size="15G",
):
name, instance = createWorker(
base_name, name, instance = createWorker(
worker_name_prefix,
worker_id,
worker_type,
@@ -143,21 +144,18 @@ def addWorker(
shm_size,
)

if name[0] not in workers:
workers[name[0]] = [name[1]]
else:
workers[name[0]].append(name[1])

workers[base_name].append(name)
c["workers"].append(instance)



# Docker workers
fqdn = os.environ["BUILDMASTER_WG_IP"]

addWorker(
"amd-bbw",
1,
"-debian-12-32-bit",
"debian-12-32-bit",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian12-386",
jobs=10,
@@ -168,7 +166,7 @@ addWorker(
addWorker(
"amd-bbw",
2,
"-debian-12-32-bit",
"debian-12-32-bit",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian12-386",
jobs=10,
@@ -179,7 +177,7 @@ addWorker(
addWorker(
"apexis-bbw",
3,
"-debian-12-32-bit",
"debian-12-32-bit",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian12-386",
jobs=10,
@@ -190,7 +188,7 @@ addWorker(
addWorker(
"apexis-bbw",
3,
"-msan-clang-debian-11",
"msan-clang-debian-11",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian11-msan",
jobs=20,
@@ -200,7 +198,7 @@ addWorker(
addWorker(
"apexis-bbw",
3,
"-ubuntu-2204-jepsen-mariadb",
"ubuntu-2204-jepsen-mariadb",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu22.04-jepsen-mariadb",
jobs=5,
53 changes: 25 additions & 28 deletions master-docker-nonstandard/master.cfg
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@ from twisted.internet import defer
import docker
import os
import sys
from collections import defaultdict

sys.path.insert(0, "/srv/buildbot/master")
sys.setrecursionlimit(10000)
@@ -121,7 +122,7 @@ c["schedulers"] = SCHEDULERS
# worker name and password must be configured on the worker.
c["workers"] = []

workers = {}
workers = defaultdict(list)


def addWorker(
@@ -133,7 +134,7 @@ def addWorker(
save_packages=False,
shm_size="15G",
):
name, instance = createWorker(
base_name, name, instance = createWorker(
worker_name_prefix,
worker_id,
worker_type,
@@ -143,11 +144,7 @@ def addWorker(
shm_size,
)

if name[0] not in workers:
workers[name[0]] = [name[1]]
else:
workers[name[0]].append(name[1])

workers[base_name].append(name)
c["workers"].append(instance)


@@ -248,7 +245,7 @@ c["workers"].append(
addWorker(
"hz-bbw",
6,
"-bigtest-ubuntu-2004",
"bigtest-ubuntu-2004",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=20,
@@ -260,7 +257,7 @@ for w_name in ["ppc64le-osuosl-bbw"]:
addWorker(
w_name,
1,
"-ubuntu-2004",
"ubuntu-2004",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=7,
@@ -270,7 +267,7 @@ for w_name in ["ppc64le-osuosl-bbw"]:
addWorker(
w_name,
1,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=30,
@@ -288,7 +285,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]:
addWorker(
w_name,
i,
"-aocc-debian-11",
"aocc-debian-11",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian11-aocc",
jobs=jobs,
@@ -297,7 +294,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]:
addWorker(
w_name,
i,
"-asan-ubuntu-2404",
"asan-ubuntu-2404",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu24.04",
jobs=jobs,
@@ -306,7 +303,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]:
addWorker(
w_name,
i,
"-icc-ubuntu-2204",
"icc-ubuntu-2204",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu22.04-icc",
jobs=jobs,
@@ -315,7 +312,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]:
addWorker(
w_name,
i,
"-ubuntu-2004",
"ubuntu-2004",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=jobs,
@@ -326,7 +323,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]:
addWorker(
"amd-bbw",
1,
"-valgrind-fedora-40",
"valgrind-fedora-40",
os.environ["CONTAINER_REGISTRY_URL"]
+ "fedora40-valgrind",
jobs=20,
@@ -335,7 +332,7 @@ addWorker(
addWorker(
"amd-bbw",
2,
"-valgrind-fedora-40",
"valgrind-fedora-40",
os.environ["CONTAINER_REGISTRY_URL"]
+ "fedora40-valgrind",
jobs=20,
@@ -344,7 +341,7 @@ addWorker(
addWorker(
"hz-bbw",
6,
"-valgrind-fedora-40",
"valgrind-fedora-40",
os.environ["CONTAINER_REGISTRY_URL"]
+ "fedora40-valgrind",
jobs=20,
@@ -354,7 +351,7 @@ addWorker(
addWorker(
"hz-bbw",
1,
"-msan-clang-16-debian-11",
"msan-clang-16-debian-11",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian11-msan-clang-16",
jobs=20,
@@ -363,7 +360,7 @@ addWorker(
addWorker(
"hz-bbw",
4,
"-msan-clang-16-debian-11",
"msan-clang-16-debian-11",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian11-msan-clang-16",
jobs=20,
@@ -372,7 +369,7 @@ addWorker(
addWorker(
"hz-bbw",
5,
"-msan-clang-16-debian-11",
"msan-clang-16-debian-11",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian11-msan-clang-16",
jobs=30,
@@ -382,7 +379,7 @@ addWorker(
addWorker(
"hz-bbw",
2,
"-debian-12",
"debian-12",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian12",
jobs=20,
@@ -391,7 +388,7 @@ addWorker(
addWorker(
"hz-bbw",
5,
"-debian-12",
"debian-12",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian12",
jobs=20,
@@ -401,7 +398,7 @@ addWorker(
addWorker(
"aarch64-bbw",
6,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=10,
@@ -410,7 +407,7 @@ addWorker(
addWorker(
"aarch64-bbw",
6,
"-debian-10-bintar",
"debian-10-bintar",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian10-bintar",
jobs=10,
@@ -420,7 +417,7 @@ addWorker(
addWorker(
"hz-bbw",
5,
"-centos-7-bintar",
"centos-7-bintar",
os.environ["CONTAINER_REGISTRY_URL"]
+ "centos7-bintar",
jobs=10,
@@ -430,7 +427,7 @@ addWorker(
addWorker(
"s390x-bbw",
1,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=7,
@@ -440,7 +437,7 @@ addWorker(
addWorker(
"s390x-bbw",
2,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=7,
@@ -450,7 +447,7 @@ addWorker(
addWorker(
"s390x-bbw",
3,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=7,
13 changes: 5 additions & 8 deletions master-galera/master.cfg
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@ import docker
import yaml

from datetime import timedelta
from collections import defaultdict

sys.setrecursionlimit(10000)

@@ -169,7 +170,7 @@ c["workers"].append(
# Docker workers
GALERA_PACKAGES = os.environ["GALERA_PACKAGES_DIR"]

workers = {}
workers = defaultdict(list)

def addWorker(
worker_name_prefix,
@@ -180,7 +181,7 @@ def addWorker(
save_packages=False,
shm_size="15G",
):
name, instance = createWorker(
base_name, name, instance = createWorker(
worker_name_prefix,
worker_id,
worker_type,
@@ -196,11 +197,7 @@ def addWorker(
],
)

if name[0] not in workers:
workers[name[0]] = [name[1]]
else:
workers[name[0]].append(name[1])

workers[base_name].append(name)
c["workers"].append(instance)


@@ -251,7 +248,7 @@ for platform in ALL_PLATFORMS:
addWorker(
w_name,
i,
"-" + os_name,
os_name,
quay_name,
jobs=jobs,
save_packages=True,
49 changes: 23 additions & 26 deletions master-protected-branches/master.cfg
Original file line number Diff line number Diff line change
@@ -1,28 +1,29 @@
# -*- python -*-
# ex: set filetype=python:

import os
import sys
from collections import defaultdict
from datetime import timedelta

import docker
from buildbot.plugins import *
from buildbot.process.properties import Property, Properties
from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand
from buildbot.process.properties import Properties, Property
from buildbot.process.remotecommand import RemoteCommand
from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver
from buildbot.steps.shell import Compile, SetPropertyFromCommand, ShellCommand, Test
from buildbot.steps.source.github import GitHub
from buildbot.process.remotecommand import RemoteCommand
from datetime import timedelta
from twisted.internet import defer

import docker
import os
import sys

sys.setrecursionlimit(10000)

sys.path.insert(0, "/srv/buildbot/master")

from common_factories import *
from constants import *
from utils import *
from locks import *
from schedulers_definition import SCHEDULERS
from common_factories import *
from utils import *

cfg_dir = os.path.abspath(os.path.dirname(__file__))

@@ -157,7 +158,7 @@ c["workers"].append(
)
)

workers = {}
workers = defaultdict(list)


def addWorker(
@@ -169,7 +170,7 @@ def addWorker(
save_packages=False,
shm_size="15G",
):
name, instance = createWorker(
base_name, name, worker_instance = createWorker(
worker_name_prefix,
worker_id,
worker_type,
@@ -179,12 +180,8 @@ def addWorker(
shm_size,
)

if name[0] not in workers:
workers[name[0]] = [name[1]]
else:
workers[name[0]].append(name[1])

c["workers"].append(instance)
workers[base_name].append(name)
c["workers"].append(worker_instance)


for w_name in ["hz-bbw"]:
@@ -193,63 +190,63 @@ for w_name in ["hz-bbw"]:
addWorker(
w_name,
i,
"-debian-11-debug-ps-embed",
"debian-11-debug-ps-embed",
os.environ["CONTAINER_REGISTRY_URL"] + "debian11",
jobs=14,
save_packages=False,
)
addWorker(
w_name,
i,
"-debian-12",
"debian-12",
os.environ["CONTAINER_REGISTRY_URL"] + "debian12",
jobs=jobs,
save_packages=True,
)
addWorker(
w_name,
i,
"-debian-12-debug-embed",
"debian-12-debug-embed",
os.environ["CONTAINER_REGISTRY_URL"] + "debian12",
jobs=14,
save_packages=False,
)
addWorker(
w_name,
i,
"-fedora-40",
"fedora-40",
os.environ["CONTAINER_REGISTRY_URL"] + "fedora40",
jobs=jobs,
save_packages=True,
)
addWorker(
w_name,
i,
"-last-N-failed",
"last-N-failed",
os.environ["CONTAINER_REGISTRY_URL"] + "rhel9",
jobs=jobs,
save_packages=True,
)
addWorker(
w_name,
i,
"-ubuntu-2004-clang",
"ubuntu-2004-clang",
"vladbogo/bb:amd64-ubuntu-2004-clang",
jobs=jobs,
save_packages=True,
)
addWorker(
w_name,
i,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04",
jobs=14,
save_packages=True,
)
addWorker(
w_name,
i,
"-ubuntu-2204-debug-ps",
"ubuntu-2204-debug-ps",
os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu22.04",
jobs=14,
save_packages=False,
40 changes: 9 additions & 31 deletions master.cfg
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@
# ex: set filetype=python:

import os
from collections import defaultdict

import yaml

@@ -130,35 +131,7 @@ c["workers"] = []

# Docker workers

workers = {}


def addWorker(
worker_name_prefix,
worker_id,
worker_type,
dockerfile,
jobs=5,
save_packages=False,
shm_size="15G",
):
name, instance = createWorker(
worker_name_prefix,
worker_id,
worker_type,
dockerfile,
jobs,
save_packages,
shm_size,
)

if name[0] not in workers:
workers[name[0]] = [name[1]]
else:
workers[name[0]].append(name[1])

c["workers"].append(instance)

workers = defaultdict(list)

for w_name in master_config["workers"]:
jobs = 7
@@ -184,15 +157,20 @@ for w_name in master_config["workers"]:
if builder.startswith("x86"):
os_name += "-i386"
quay_name += "-386"
addWorker(

base_name, name, worker_instance = createWorker(
worker_name,
worker_id,
"-" + os_name,
os_name,
quay_name,
jobs=jobs,
save_packages=True,
shm_size="15G",
)

workers[base_name].append(name)
c["workers"].append(worker_instance)

####### FACTORY CODE

f_quick_build = getQuickBuildFactory("nm", mtrDbPool)
37 changes: 17 additions & 20 deletions utils.py
Original file line number Diff line number Diff line change
@@ -79,25 +79,22 @@ def createWorker(
"/srv/buildbot/packages:/mnt/packages",
MASTER_PACKAGES + "/:/packages",
],
) -> Tuple[str, worker.DockerLatentWorker]:
) -> Tuple[str, str, worker.DockerLatentWorker]:
worker_name = f"{worker_name_prefix}{worker_id}-docker"
name = f"{worker_name}{worker_type}"

if worker_name_prefix.startswith("hz"):
b_name = "x64-bbw"
elif worker_name_prefix.startswith("intel"):
b_name = "x64-bbw"
elif worker_name_prefix.startswith("ppc64le"):
b_name = "ppc64le-bbw"
elif worker_name_prefix.startswith("amd"):
b_name = "x64-bbw"
elif worker_name_prefix.startswith("apexis"):
b_name = "x64-bbw"
elif worker_name_prefix.startswith("ns"):
b_name = "x64-bbw"
else:
b_name = worker_name_prefix
base_name = b_name + "-docker" + worker_type
name = f"{worker_name}{worker_type}{worker_name_suffix}"

# TODO(cvicentiu) Remove this list when refactoring YAML.
b_name = worker_name_prefix
X64_BUILDER_PREFIXES = ["hz", "intel", "amd", "apexis", "ns"]
PPC64LE_BUILDER_PREFIXES = ["ppc64le"]
for x64_prefix in X64_BUILDER_PREFIXES:
if worker_name_prefix.startswith(x64_prefix):
b_name = "x64-bbw"
for ppc_prefix in PPC64LE_BUILDER_PREFIXES:
if worker_name_prefix.startswith(ppc_prefix):
b_name = "ppc64le-bbw"

base_name = f"{b_name}-docker-{worker_type}"

# Set master FQDN - default to wireguard interface
fqdn = os.environ["BUILDMASTER_WG_IP"]
@@ -113,7 +110,7 @@ def createWorker(
need_pull = False

worker_instance = worker.DockerLatentWorker(
name + worker_name_suffix,
name,
None,
docker_host=private_config["private"]["docker_workers"][worker_name],
image=image_str,
@@ -135,7 +132,7 @@ def createWorker(
volumes=volumes,
properties={"jobs": jobs, "save_packages": save_packages},
)
return ((base_name, name + worker_name_suffix), worker_instance)
return (base_name, name, worker_instance)


def printEnv() -> steps.ShellCommand: