refactor: remove I18N binary file django.mo and compile the file before starting processes

.gitignore

@@ -46,7 +46,7 @@ coverage.xml
 .hypothesis/
 
 # Translations
-# *.mo
+*.mo
 *.pot
 
 # Django stuff:

apiserver/Dockerfile

@@ -67,6 +67,23 @@ RUN mkdir -p ./public
 COPY --from=admin42 /build/paasng/public ./public
 ADD ./paasng .
 
+
+# Add extra files: static assets & I18N .mo translation file. 
+# the "BKKRILL_ENCRYPT_SECRET_KEY" settings is required for starting the project,
+# set it to a non-empty value to pass the check.
+ENV PAAS_BKKRILL_ENCRYPT_SECRET_KEY='none'
+
+# Some edition might need to add extra apps, do it by set the arg during the build process,
+# e.g. docker build . --build-arg EXTRA_INSTALLED_APPS="['foo']" ...
+ARG EXTRA_INSTALLED_APPS
+ENV PAAS_EXTRA_INSTALLED_APPS=${EXTRA_INSTALLED_APPS}
+
+RUN poetry run python manage.py collectstatic --noinput
+# "gettext" package is required for running "compilemessages"
+RUN apt-get install -y gettext
+RUN poetry run python manage.py compilemessages
+
+
 COPY ./start.sh .
 COPY ./start_celery.sh .
 COPY ./init.sh .

apiserver/dev_utils/unittest/Dockerfile.devops.unittest

@@ -15,7 +15,7 @@ ADD ./.babelrc .
 ADD ./.bootstraprc .
 ADD ./.eslintrc.js .
 ADD ./postcss.config.js .
-ADD ./paasng ./paasng/
+ADD ./paasng/assets ./paasng/assets
 ADD ./configs ./configs
 
 # 执行编译
@@ -61,9 +61,26 @@ RUN poetry config virtualenvs.create false && poetry config experimental.new-ins
 WORKDIR /app
 ADD ./paasng ./paasng
 RUN mkdir -p ./public
-COPY --from=Admin42 /build/paasng/public ./public
+COPY --from=Admin42 /build/paasng/public ./paasng/public
 
 WORKDIR /app/paasng
+
+# Add extra files: static assets & I18N .mo translation file. 
+# the "BKKRILL_ENCRYPT_SECRET_KEY" settings is required for starting the project,
+# set it to a non-empty value to pass the check.
+ENV PAAS_BKKRILL_ENCRYPT_SECRET_KEY='none'
+
+# Some edition might need to add extra apps, do it by set the arg during the build process,
+# e.g. docker build . --build-arg EXTRA_INSTALLED_APPS="['foo']" ...
+ARG EXTRA_INSTALLED_APPS
+ENV PAAS_EXTRA_INSTALLED_APPS=${EXTRA_INSTALLED_APPS}
+
+RUN poetry run python manage.py collectstatic --noinput
+# "gettext" package is required for running "compilemessages"
+RUN apt-get install -y gettext
+RUN poetry run python manage.py compilemessages
+
+
 ADD ./start.sh .
 COPY ./start_celery.sh .
 CMD ["./start.sh"]

apiserver/paasng/paasng/misc/metrics/basic_services/mysql.py

@@ -15,7 +15,9 @@
 # We undertake not to change the open source license (MIT license) applicable
 # to the current version of the project delivered to anyone in the future.
 
-from blue_krill.monitoring.probe.base import ProbeSet
+from typing import List, Type
+
+from blue_krill.monitoring.probe.base import ProbeSet, VirtualProbe
 
 from paasng.misc.monitoring.healthz.probes import PlatformMysqlProbe, WorkloadsMysqlProbe
 
@@ -28,5 +30,12 @@ class MySQLAvailableMetric(GaugeMetric):
 
     @classmethod
     def calc_value(cls) -> bool:
-        probe_set = ProbeSet([PlatformMysqlProbe, WorkloadsMysqlProbe])
+        probe_types: List[Type[VirtualProbe]] = []
+        # These probe might use an empty config object if the database config is absent
+        if PlatformMysqlProbe.config.host:
+            probe_types.append(PlatformMysqlProbe)
+        if WorkloadsMysqlProbe.config.host:
+            probe_types.append(WorkloadsMysqlProbe)
+
+        probe_set = ProbeSet(probe_types)
         return not probe_set.examination().is_death

apiserver/paasng/paasng/misc/monitoring/healthz/probes.py

@@ -19,7 +19,7 @@
 
 from blue_krill.monitoring.probe.base import Issue, VirtualProbe
 from blue_krill.monitoring.probe.http import BKHttpProbe, HttpProbe
-from blue_krill.monitoring.probe.mysql import MySQLProbe, transfer_django_db_settings
+from blue_krill.monitoring.probe.mysql import MySQLConfig, MySQLProbe, transfer_django_db_settings
 from blue_krill.monitoring.probe.redis import RedisProbe, RedisSentinelProbe
 from django.conf import settings
 from django.utils.module_loading import import_string
@@ -50,14 +50,29 @@ def __init__(self):
         self.config = transfer_django_db_settings(settings.BK_CONSOLE_DBCONF)
 
 
+# The database config might be absent when running "collectstatic" command in the
+# docker building process, skip the probe config initialization in this case or it
+# will raise an exception.
+_empty_probe_config = MySQLConfig("", 0, "", "", "")
+
+_default_mysql_probe_config = _empty_probe_config
+if (_default_db := settings.DATABASES.get("default")) and _default_db["ENGINE"] != "django.db.backends.dummy":
+    _default_mysql_probe_config = transfer_django_db_settings(_default_db)
+
+
 class PlatformMysqlProbe(MySQLProbe):
     name = "platform-mysql-ng"
-    config = transfer_django_db_settings(settings.DATABASES["default"])
+    config = _default_mysql_probe_config
+
+
+_wl_mysql_probe_config = _empty_probe_config
+if (_wl_db := settings.DATABASES.get("workloads")) and _wl_db["ENGINE"] != "django.db.backends.dummy":
+    _wl_mysql_probe_config = transfer_django_db_settings(_wl_db)
 
 
 class WorkloadsMysqlProbe(MySQLProbe):
     name = "platform-mysql-wl"
-    config = transfer_django_db_settings(settings.DATABASES["workloads"])
+    config = _wl_mysql_probe_config
 
 
 class ESBProbe(BKHttpProbe):

apiserver/paasng/paasng/settings/__init__.py

@@ -498,10 +498,15 @@ def _build_file_handler(log_path: Path, filename: str, format: str) -> Dict:
 # Django 基础配置（自定义）
 # ------------------------
 
-DATABASES = {
-    "default": get_database_conf(settings),
-    "workloads": get_database_conf(settings, encrypted_url_var="WL_DATABASE_URL", env_var_prefix="WL_"),
-}
+DATABASES = {}
+
+# When running "collectstatic" command, the database config is not available, so we
+# make it optional.
+if default_db_conf := get_database_conf(settings):
+    DATABASES["default"] = default_db_conf
+if wl_db_conf := get_database_conf(settings, encrypted_url_var="WL_DATABASE_URL", env_var_prefix="WL_"):
+    DATABASES["workloads"] = wl_db_conf
+
 DATABASE_ROUTERS = ["paasng.core.core.storages.dbrouter.WorkloadsDBRouter"]
 
 # == Redis 相关配置项，该 Redis 服务将被用于：缓存

apiserver/paasng/paasng/utils/blobstore.py

@@ -15,8 +15,8 @@
 # We undertake not to change the open source license (MIT license) applicable
 # to the current version of the project delivered to anyone in the future.
 
-"""BlobStore client
-"""
+"""BlobStore client"""
+
 import json
 import logging
 import urllib.parse
@@ -119,8 +119,8 @@ def get_storage_by_bucket(bucket: str, store_type: Optional[str] = None):
             error_code = int(e.response["Error"]["Code"])
             if error_code == 404:
                 s3.create_bucket(Bucket=bucket)
-            else:
-                raise
+        except Exception as e:
+            logger.warning("Error getting bucket %s, error: %s, further actions on it might fail.", bucket, e)
 
         return RGWBoto3Storage(
             bucket=bucket,

apiserver/start.sh

@@ -23,10 +23,6 @@ GUNICORN_LOG_LEVEL=${GUNICORN_LOG_LEVEL:-INFO}
 GUNICORN_TIMEOUT=${GUNICORN_TIMEOUT:-150}
 GUNICORN_MAX_REQUESTS=${GUNICORN_MAX_REQUESTS:-2048}
 
-## 初始化静态资源.
-mkdir -p ../public/assets
-python manage.py collectstatic
-
 command="gunicorn paasng.wsgi -w ${GUNICORN_CONCURRENCY} --timeout ${GUNICORN_TIMEOUT} -b [::]:8000 -k gevent --max-requests ${GUNICORN_MAX_REQUESTS} --access-logfile '-' --access-logformat '%(h)s %(l)s %(u)s %(t)s \"%(r)s\" %(s)s %(b)s \"%(f)s\" \"%(a)s\" in %(L)s seconds' --log-level ${GUNICORN_LOG_LEVEL} --log-file=-"
 
 ## Run!

apiserver/start_admin42.sh

@@ -33,9 +33,6 @@ fi
 
 python manage.py migrate
 
-mkdir -p ../public/assets
-python manage.py collectstatic --noinput
-
 ## Run!
 command="python manage.py runserver 0.0.0.0:"$PORT
 exec bash -c "$command"