Validate environment variables during checkconfig

Both BuildBot environments now run in containers so we can get rid of the default="" in << os.getenv >>. To check that the variable is defined and to prevent typos use os.environ which would raise a key error if the variable does not exist. PORT is a special case, because it's defined in docker-compose generation script and is not present in the .env and .env.dev files, hence the hardcoded value in the validation script. Keep os.environ for PORT because the correct value will be passed by docker-compose during initialization (docker-compose --env-file .env up)
RazvanLiviuVarzaru · Jan 9, 2025 · 8bc94d5 · 8bc94d5
1 parent 90195f6
commit 8bc94d5
Show file tree

Hide file tree

Showing 13 changed files with 149 additions and 221 deletions.
diff --git a/common_factories.py b/common_factories.py
@@ -648,7 +648,7 @@ def getRpmAutobakeFactory(mtrDbPool):
                 util.Interpolate(
                     'wget --no-check-certificate -cO ../MariaDB-shared-5.3.%(kw:arch)s.rpm "%(kw:url)s/helper_files/mariadb-shared-5.3-%(kw:arch)s.rpm" && wget -cO ../MariaDB-shared-10.1.%(kw:arch)s.rpm "%(kw:url)s/helper_files/mariadb-shared-10.1-kvm-rpm-%(kw:rpm_type)s-%(kw:arch)s.rpm"',
                     arch=getArch,
-                    url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"),
+                    url=os.environ["ARTIFACTS_URL"],
                     rpm_type=util.Property("rpm_type"),
                 ),
             ],
@@ -736,7 +736,7 @@ def getRpmAutobakeFactory(mtrDbPool):
                 echo "module_hotfixes = 1" >> MariaDB.repo
             fi
         """,
-                    url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"),
+                    url=os.environ["ARTIFACTS_URL"],
                 ),
             ]
         )
@@ -761,7 +761,7 @@ def getRpmAutobakeFactory(mtrDbPool):
             descriptionDone=util.Interpolate(
                 """
 Repository available with: curl %(kw:url)s/%(prop:tarbuildnum)s/%(prop:buildername)s/MariaDB.repo -o /etc/yum.repos.d/MariaDB.repo""",
-                url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"),
+                url=os.environ["ARTIFACTS_URL"],
             ),
         )
     )

diff --git a/constants.py b/constants.py
@@ -45,7 +45,7 @@
     "amd64-ubuntu-2004-eco-php",
 ]
 
-if os.getenv("ENVIRON") == "DEV":
+if os.environ["ENVIRON"] == "DEV":
     BUILDERS_WORDPRESS = ["amd64-rhel9-wordpress"]
     BUILDERS_DOCKERLIBRARY = ["amd64-rhel9-dockerlibrary"]
 else:

diff --git a/master-bintars/master.cfg b/master-bintars/master.cfg
@@ -24,7 +24,7 @@ from locks import *
 from schedulers_definition import SCHEDULERS
 from utils import *
 
-FQDN = os.getenv("BUILDMASTER_WG_IP", default="100.64.100.1")
+FQDN = os.environ["BUILDMASTER_WG_IP"]
 
 # This is the dictionary that the buildmaster pays attention to. We also use
 # a shorter alias to save typing.
@@ -55,32 +55,29 @@ gs = reporters.GitHubStatusPush(
 c["services"].append(gs)
 c["secretsProviders"] = [
     secrets.SecretInAFile(
-        dirname=os.getenv(
-            "MASTER_CREDENTIALS_DIR",
-            default="/srv/buildbot/master/master-credential-provider",
-        )
+        dirname=os.environ["MASTER_CREDENTIALS_DIR"]
     )
 ]
 
 ####### PROJECT IDENTITY
 
 # the 'title' string will appear at the top of this buildbot installation's
 # home pages (linked to the 'titleURL').
-c["title"] = os.getenv("TITLE", default="MariaDB CI")
-c["titleURL"] = os.getenv("TITLE_URL", default="https://github.com/MariaDB/server")
+c["title"] = os.environ["TITLE"]
+c["titleURL"] = os.environ["TITLE_URL"]
 
 # the 'buildbotURL' string should point to the location where the buildbot's
 # internal web server is visible. This typically uses the port number set in
 # the 'www' entry below, but with an externally-visible host name which the
 # buildbot cannot figure out without some help.
-c["buildbotURL"] = os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/")
+c["buildbotURL"] = os.environ["BUILDMASTER_URL"]
 
 # 'protocols' contains information about protocols which master will use for
 # communicating with workers. You must define at least 'port' option that workers
 # could connect to your master with this protocol.
 # 'port' must match the value configured into the workers (with their
 # --master option)
-port = int(os.getenv("MASTER_NONLATENT_BINTARS_WORKER_PORT", default="10002"))
+port = int(os.environ["MASTER_NONLATENT_BINTARS_WORKER_PORT"])
 c["protocols"] = {"pb": {"port": port}}
 
 ####### DB URL
@@ -117,22 +114,8 @@ def mkWorker(name, **kwargs):
 # a Worker object, specifying a unique worker name and password.  The same
 # worker name and password must be configured on the worker.
 
-# default_workers_bintar exists because master-bintar in production
-# is a standolone service, not a container sourcing .env
-default_workers_bintar = """
-{
-    "ro-apexis-bbw03-x64": {
-        "max_builds": 2,
-        "jobs": 12
-    },
-    "bg-bbw1-x64": {
-        "max_builds": 1,
-        "jobs": 12
-    }
-}
-"""
 workers_bintar = json.loads(
-    os.getenv("MASTER_NONLATENT_BINTARS_WORKERS", default=default_workers_bintar)
+    os.environ["MASTER_NONLATENT_BINTARS_WORKERS"]
 )
 
 c["workers"] = []
@@ -356,7 +339,7 @@ builder_definitions = {
     "centos-6": "centos6",
 }
 
-current_port = int(os.getenv("MASTER_NONLATENT_BINTARS_VM_PORT", default=10000))
+current_port = int(os.environ["MASTER_NONLATENT_BINTARS_VM_PORT"])
 
 for b in builder_definitions:
     for arch in ["amd64", "i386"]:
@@ -388,7 +371,7 @@ c["multiMaster"] = True
 
 c["mq"] = {  # Need to enable multimaster aware mq. Wamp is the only option for now.
     "type": "wamp",
-    "router_url": os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"),
+    "router_url": os.environ["MQ_ROUTER_URL"],
     "realm": "realm1",
     # valid are: none, critical, error, warn, info, debug, trace
     "wamp_debug_level": "info",

diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg
@@ -23,7 +23,7 @@ from locks import *
 from schedulers_definition import SCHEDULERS
 from utils import *
 
-FQDN = os.getenv("BUILDMASTER_WG_IP", default="100.64.100.1")
+FQDN = os.environ["BUILDMASTER_WG_IP"]
 
 # This is the dictionary that the buildmaster pays attention to. We also use
 # a shorter alias to save typing.
@@ -54,31 +54,28 @@ gs = reporters.GitHubStatusPush(
 c["services"].append(gs)
 c["secretsProviders"] = [
     secrets.SecretInAFile(
-        dirname=os.getenv(
-            "MASTER_CREDENTIALS_DIR",
-            default="/srv/buildbot/master/master-credential-provider",
-        )
+        dirname=os.environ["MASTER_CREDENTIALS_DIR"]
     )
 ]
 ####### PROJECT IDENTITY
 
 # the 'title' string will appear at the top of this buildbot installation's
 # home pages (linked to the 'titleURL').
-c["title"] = os.getenv("TITLE", default="MariaDB CI")
-c["titleURL"] = os.getenv("TITLE_URL", default="https://github.com/MariaDB/server")
+c["title"] = os.environ["TITLE"]
+c["titleURL"] = os.environ["TITLE_URL"]
 
 # the 'buildbotURL' string should point to the location where the buildbot's
 # internal web server is visible. This typically uses the port number set in
 # the 'www' entry below, but with an externally-visible host name which the
 # buildbot cannot figure out without some help.
-c["buildbotURL"] = os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/")
+c["buildbotURL"] = os.environ["BUILDMASTER_URL"]
 
 # 'protocols' contains information about protocols which master will use for
 # communicating with workers. You must define at least 'port' option that workers
 # could connect to your master with this protocol.
 # 'port' must match the value configured into the workers (with their
 # --master option)
-port = int(os.getenv("PORT", default="10001"))
+port = int(os.environ["PORT"])
 c["protocols"] = {"pb": {"port": port}}
 
 ####### DB URL
@@ -142,13 +139,13 @@ def addWorker(
 
 
 # Docker workers
-fqdn = os.getenv("BUILDMASTER_WG_IP", default="100.64.100.1")
+fqdn = os.environ["BUILDMASTER_WG_IP"]
 
 addWorker(
     "amd-bbw",
     1,
     "-debian-12-32-bit",
-    os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:")
+    os.environ["CONTAINER_REGISTRY_URL"]
     + "debian12-386",
     jobs=10,
     save_packages=False,
@@ -159,7 +156,7 @@ addWorker(
     "amd-bbw",
     2,
     "-debian-12-32-bit",
-    os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:")
+    os.environ["CONTAINER_REGISTRY_URL"]
     + "debian12-386",
     jobs=10,
     save_packages=False,
@@ -170,7 +167,7 @@ addWorker(
     "apexis-bbw",
     3,
     "-debian-12-32-bit",
-    os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:")
+    os.environ["CONTAINER_REGISTRY_URL"]
     + "debian12-386",
     jobs=10,
     save_packages=False,
@@ -181,7 +178,7 @@ addWorker(
     "apexis-bbw",
     3,
     "-msan-clang-debian-11",
-    os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:")
+    os.environ["CONTAINER_REGISTRY_URL"]
     + "debian11-msan",
     jobs=20,
     save_packages=False,
@@ -191,7 +188,7 @@ addWorker(
     "apexis-bbw",
     3,
     "-ubuntu-2204-jepsen-mariadb",
-    os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:")
+    os.environ["CONTAINER_REGISTRY_URL"]
     + "ubuntu22.04-jepsen-mariadb",
     jobs=5,
     save_packages=False,
@@ -805,7 +802,7 @@ f_eco_php.addStep(
         command=[
             "/buildbot/installdb.sh",
             util.Interpolate(
-                os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")
+                os.environ["ARTIFACTS_URL"]
                 + "/%(prop:tarbuildnum)s/%(prop:parentbuildername)s/%(prop:mariadb_binary)s"
             ),
             "--plugin-load-add=auth_pam",
@@ -862,7 +859,7 @@ f_eco_pymysql.addStep(
         command=[
             "/buildbot/installdb.sh",
             util.Interpolate(
-                os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")
+                os.environ["ARTIFACTS_URL"]
                 + "/%(prop:tarbuildnum)s/%(prop:parentbuildername)s/%(prop:mariadb_binary)s"
             ),
         ],
@@ -908,7 +905,7 @@ f_eco_mysqljs.addStep(
         command=[
             "/buildbot/installdb.sh",
             util.Interpolate(
-                os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")
+                os.environ["ARTIFACTS_URL"]
                 + "/%(prop:tarbuildnum)s/%(prop:parentbuildername)s/%(prop:mariadb_binary)s"
             ),
         ],
@@ -1167,7 +1164,7 @@ c["multiMaster"] = True
 
 c["mq"] = {  # Need to enable multimaster aware mq. Wamp is the only option for now.
     "type": "wamp",
-    "router_url": os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"),
+    "router_url": os.environ["MQ_ROUTER_URL"],
     "realm": "realm1",
     # valid are: none, critical, error, warn, info, debug, trace
     "wamp_debug_level": "info",