diff --git a/.gitignore b/.gitignore index c11e027185..f784b54c95 100644 --- a/.gitignore +++ b/.gitignore @@ -46,7 +46,7 @@ coverage.xml .hypothesis/ # Translations -# *.mo +*.mo *.pot # Django stuff: diff --git a/apiserver/Dockerfile b/apiserver/Dockerfile index 2983675190..3a2a80b0d5 100644 --- a/apiserver/Dockerfile +++ b/apiserver/Dockerfile @@ -67,6 +67,23 @@ RUN mkdir -p ./public COPY --from=admin42 /build/paasng/public ./public ADD ./paasng . + +# Add extra files: static assets & I18N .mo translation file. +# the "BKKRILL_ENCRYPT_SECRET_KEY" settings is required for starting the project, +# set it to a non-empty value to pass the check. +ENV PAAS_BKKRILL_ENCRYPT_SECRET_KEY='none' + +# Some edition might need to add extra apps, do it by set the arg during the build process, +# e.g. docker build . --build-arg EXTRA_INSTALLED_APPS="['foo']" ... +ARG EXTRA_INSTALLED_APPS +ENV PAAS_EXTRA_INSTALLED_APPS=${EXTRA_INSTALLED_APPS} + +RUN poetry run python manage.py collectstatic --noinput +# "gettext" package is required for running "compilemessages" +RUN apt-get install -y gettext +RUN poetry run python manage.py compilemessages + + COPY ./start.sh . COPY ./start_celery.sh . COPY ./init.sh . diff --git a/apiserver/dev_utils/unittest/Dockerfile.devops.unittest b/apiserver/dev_utils/unittest/Dockerfile.devops.unittest index a80e6a8842..d4ffaefc1d 100644 --- a/apiserver/dev_utils/unittest/Dockerfile.devops.unittest +++ b/apiserver/dev_utils/unittest/Dockerfile.devops.unittest @@ -15,7 +15,7 @@ ADD ./.babelrc . ADD ./.bootstraprc . ADD ./.eslintrc.js . ADD ./postcss.config.js . -ADD ./paasng ./paasng/ +ADD ./paasng/assets ./paasng/assets ADD ./configs ./configs # 执行编译 @@ -61,9 +61,26 @@ RUN poetry config virtualenvs.create false && poetry config experimental.new-ins WORKDIR /app ADD ./paasng ./paasng RUN mkdir -p ./public -COPY --from=Admin42 /build/paasng/public ./public +COPY --from=Admin42 /build/paasng/public ./paasng/public WORKDIR /app/paasng + +# Add extra files: static assets & I18N .mo translation file. +# the "BKKRILL_ENCRYPT_SECRET_KEY" settings is required for starting the project, +# set it to a non-empty value to pass the check. +ENV PAAS_BKKRILL_ENCRYPT_SECRET_KEY='none' + +# Some edition might need to add extra apps, do it by set the arg during the build process, +# e.g. docker build . --build-arg EXTRA_INSTALLED_APPS="['foo']" ... +ARG EXTRA_INSTALLED_APPS +ENV PAAS_EXTRA_INSTALLED_APPS=${EXTRA_INSTALLED_APPS} + +RUN poetry run python manage.py collectstatic --noinput +# "gettext" package is required for running "compilemessages" +RUN apt-get install -y gettext +RUN poetry run python manage.py compilemessages + + ADD ./start.sh . COPY ./start_celery.sh . CMD ["./start.sh"] diff --git a/apiserver/paasng/locale/en/LC_MESSAGES/django.mo b/apiserver/paasng/locale/en/LC_MESSAGES/django.mo deleted file mode 100644 index 579f0ebe79..0000000000 Binary files a/apiserver/paasng/locale/en/LC_MESSAGES/django.mo and /dev/null differ diff --git a/apiserver/paasng/paasng/misc/metrics/basic_services/mysql.py b/apiserver/paasng/paasng/misc/metrics/basic_services/mysql.py index c3932c5642..9a352a401e 100644 --- a/apiserver/paasng/paasng/misc/metrics/basic_services/mysql.py +++ b/apiserver/paasng/paasng/misc/metrics/basic_services/mysql.py @@ -15,7 +15,9 @@ # We undertake not to change the open source license (MIT license) applicable # to the current version of the project delivered to anyone in the future. -from blue_krill.monitoring.probe.base import ProbeSet +from typing import List, Type + +from blue_krill.monitoring.probe.base import ProbeSet, VirtualProbe from paasng.misc.monitoring.healthz.probes import PlatformMysqlProbe, WorkloadsMysqlProbe @@ -28,5 +30,12 @@ class MySQLAvailableMetric(GaugeMetric): @classmethod def calc_value(cls) -> bool: - probe_set = ProbeSet([PlatformMysqlProbe, WorkloadsMysqlProbe]) + probe_types: List[Type[VirtualProbe]] = [] + # These probe might use an empty config object if the database config is absent + if PlatformMysqlProbe.config.host: + probe_types.append(PlatformMysqlProbe) + if WorkloadsMysqlProbe.config.host: + probe_types.append(WorkloadsMysqlProbe) + + probe_set = ProbeSet(probe_types) return not probe_set.examination().is_death diff --git a/apiserver/paasng/paasng/misc/monitoring/healthz/probes.py b/apiserver/paasng/paasng/misc/monitoring/healthz/probes.py index a99ee3c6e0..ac2010d0b5 100644 --- a/apiserver/paasng/paasng/misc/monitoring/healthz/probes.py +++ b/apiserver/paasng/paasng/misc/monitoring/healthz/probes.py @@ -19,7 +19,7 @@ from blue_krill.monitoring.probe.base import Issue, VirtualProbe from blue_krill.monitoring.probe.http import BKHttpProbe, HttpProbe -from blue_krill.monitoring.probe.mysql import MySQLProbe, transfer_django_db_settings +from blue_krill.monitoring.probe.mysql import MySQLConfig, MySQLProbe, transfer_django_db_settings from blue_krill.monitoring.probe.redis import RedisProbe, RedisSentinelProbe from django.conf import settings from django.utils.module_loading import import_string @@ -50,14 +50,29 @@ def __init__(self): self.config = transfer_django_db_settings(settings.BK_CONSOLE_DBCONF) +# The database config might be absent when running "collectstatic" command in the +# docker building process, skip the probe config initialization in this case or it +# will raise an exception. +_empty_probe_config = MySQLConfig("", 0, "", "", "") + +_default_mysql_probe_config = _empty_probe_config +if (_default_db := settings.DATABASES.get("default")) and _default_db["ENGINE"] != "django.db.backends.dummy": + _default_mysql_probe_config = transfer_django_db_settings(_default_db) + + class PlatformMysqlProbe(MySQLProbe): name = "platform-mysql-ng" - config = transfer_django_db_settings(settings.DATABASES["default"]) + config = _default_mysql_probe_config + + +_wl_mysql_probe_config = _empty_probe_config +if (_wl_db := settings.DATABASES.get("workloads")) and _wl_db["ENGINE"] != "django.db.backends.dummy": + _wl_mysql_probe_config = transfer_django_db_settings(_wl_db) class WorkloadsMysqlProbe(MySQLProbe): name = "platform-mysql-wl" - config = transfer_django_db_settings(settings.DATABASES["workloads"]) + config = _wl_mysql_probe_config class ESBProbe(BKHttpProbe): diff --git a/apiserver/paasng/paasng/settings/__init__.py b/apiserver/paasng/paasng/settings/__init__.py index bb0aa82088..ce68887015 100644 --- a/apiserver/paasng/paasng/settings/__init__.py +++ b/apiserver/paasng/paasng/settings/__init__.py @@ -498,10 +498,15 @@ def _build_file_handler(log_path: Path, filename: str, format: str) -> Dict: # Django 基础配置(自定义) # ------------------------ -DATABASES = { - "default": get_database_conf(settings), - "workloads": get_database_conf(settings, encrypted_url_var="WL_DATABASE_URL", env_var_prefix="WL_"), -} +DATABASES = {} + +# When running "collectstatic" command, the database config is not available, so we +# make it optional. +if default_db_conf := get_database_conf(settings): + DATABASES["default"] = default_db_conf +if wl_db_conf := get_database_conf(settings, encrypted_url_var="WL_DATABASE_URL", env_var_prefix="WL_"): + DATABASES["workloads"] = wl_db_conf + DATABASE_ROUTERS = ["paasng.core.core.storages.dbrouter.WorkloadsDBRouter"] # == Redis 相关配置项,该 Redis 服务将被用于:缓存 diff --git a/apiserver/paasng/paasng/utils/blobstore.py b/apiserver/paasng/paasng/utils/blobstore.py index 9a1407cbeb..b2212d48b5 100644 --- a/apiserver/paasng/paasng/utils/blobstore.py +++ b/apiserver/paasng/paasng/utils/blobstore.py @@ -15,8 +15,8 @@ # We undertake not to change the open source license (MIT license) applicable # to the current version of the project delivered to anyone in the future. -"""BlobStore client -""" +"""BlobStore client""" + import json import logging import urllib.parse @@ -119,8 +119,8 @@ def get_storage_by_bucket(bucket: str, store_type: Optional[str] = None): error_code = int(e.response["Error"]["Code"]) if error_code == 404: s3.create_bucket(Bucket=bucket) - else: - raise + except Exception as e: + logger.warning("Error getting bucket %s, error: %s, further actions on it might fail.", bucket, e) return RGWBoto3Storage( bucket=bucket, diff --git a/apiserver/start.sh b/apiserver/start.sh index c6f0da4681..53bb89e58d 100755 --- a/apiserver/start.sh +++ b/apiserver/start.sh @@ -23,10 +23,6 @@ GUNICORN_LOG_LEVEL=${GUNICORN_LOG_LEVEL:-INFO} GUNICORN_TIMEOUT=${GUNICORN_TIMEOUT:-150} GUNICORN_MAX_REQUESTS=${GUNICORN_MAX_REQUESTS:-2048} -## 初始化静态资源. -mkdir -p ../public/assets -python manage.py collectstatic - command="gunicorn paasng.wsgi -w ${GUNICORN_CONCURRENCY} --timeout ${GUNICORN_TIMEOUT} -b [::]:8000 -k gevent --max-requests ${GUNICORN_MAX_REQUESTS} --access-logfile '-' --access-logformat '%(h)s %(l)s %(u)s %(t)s \"%(r)s\" %(s)s %(b)s \"%(f)s\" \"%(a)s\" in %(L)s seconds' --log-level ${GUNICORN_LOG_LEVEL} --log-file=-" ## Run! diff --git a/apiserver/start_admin42.sh b/apiserver/start_admin42.sh index bc99f09eb2..b9f97af7d5 100755 --- a/apiserver/start_admin42.sh +++ b/apiserver/start_admin42.sh @@ -33,9 +33,6 @@ fi python manage.py migrate -mkdir -p ../public/assets -python manage.py collectstatic --noinput - ## Run! command="python manage.py runserver 0.0.0.0:"$PORT exec bash -c "$command"