branch-3.0: [feat](doris compose) Add extra hosts option for up command #51098 (#51183)

Cherry-picked from #51098

Co-authored-by: yujun <yujun@selectdb.com>

Author: github-actions[bot]

docker/runtime/doris-compose/Dockerfile

@@ -22,7 +22,14 @@
 # docker build -f docker/runtime/doris-compose/Dockerfile -t <your-image-name>:<version> .
 
 # choose a base image
+# doris 2.1, 3.0+, master use JDK 17
 ARG JDK_IMAGE=openjdk:17-jdk-slim
+
+# doris 2.0 use JDK 8
+# build 2.0 image, example:
+# docker build --build-arg JDK_IMAGE=openjdk:8u342-jdk \
+#     -f docker/runtime/doris-compose/Dockerfile \
+#     -t <your-image-name>:<version> .
 #ARG JDK_IMAGE=openjdk:8u342-jdk
 
 # user can download a doris release package, extract it, then build its image used arg `OUTPUT_PATH`
@@ -48,7 +55,7 @@ FROM ${JDK_IMAGE}
 # set environment variables
 ENV JACOCO_VERSION=0.8.8
 
-RUN sed -i s@/deb.debian.org/@/mirrors.aliyun.com/@g /etc/apt/sources.list \
+RUN sed -i -e s@/deb.debian.org/@/mirrors.aliyun.com/@g -e s@/security.debian.org/@/mirrors.aliyun.com/@g /etc/apt/sources.list \
     && apt-get clean \
     && apt-get update \
     && apt-get install -y --no-install-recommends \

docker/runtime/doris-compose/Readme.md

@@ -49,8 +49,39 @@ If build doris use `sh build.sh --fe --be --cloud` **without do any change on th
 docker build -f docker/runtime/doris-compose/Dockerfile -t <image> .
 ```
 
+The Dockerfile default use JDK 17, for doris 2.1, 3.0, master, they all default use JDK 17.
+
+But doris 2.0 still use JDK 8, for build 2.0 image, user need specific use JDK 8 with arg `JDK_IMAGE=openjdk:8u342-jdk`. Here is build 2.0 image command:
+
+
+```shell
+docker build -f docker/runtime/doris-compose/Dockerfile \
+     --build-arg JDK_IMAGE=openjdk:8u342-jdk            \
+    -t <image> .
+```
+
+
 The `<image>` is the name you want the docker image to have.
 
+User can also download a doris release package from [Doris Home](https://doris.apache.org/docs/releasenotes/all-release) or [Doris Github](https://github.com/apache/doris/releases), extract it, then build its image with arg `OUTPUT_PATH`
+
+for example:
+
+ ```shell
+cd ~/tmp
+wget https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-3.0.5-bin-x64.tar.gz
+tar xvf apache-doris-3.0.5-bin-x64.tar.gz  # after extract, there will be a directory ./apache-doris-3.0.5-bin-x64/{fe, be, ms}
+
+# -f: the Dockerfile file
+# -t: the builded image
+# . : current directory, here it's ~/tmp, then output path is ~/tmp/apache-doris-3.0.5-bin-x64
+docker build \
+     --build-arg OUTPUT_PATH=./apache-doris-3.0.5-bin-x64 \
+     -f ~/workspace/doris/docker/runtime/doris-compose/Dockerfile \
+     -t my-doris:v3.0.5 \
+     .
+```
+
 ### 3. Install the dependent python library in 'docker/runtime/doris-compose/requirements.txt'
 
 `PyYAML` of certain version not always fit other libraries' requirements. So we suggest to use a individual environment using `venv` or `conda`.

docker/runtime/doris-compose/cluster.py

@@ -145,8 +145,7 @@ def gen_subnet_prefix16():
 
 
 def get_master_fe_endpoint(cluster_name, wait_master_fe_query_addr_file=False):
-    cluster_path = get_cluster_path(cluster_name)
-    if os.path.exists(cluster_path):
+    if os.path.exists(Cluster._get_meta_file(cluster_name)):
         master_fe_query_addr_file = get_master_fe_addr_path(cluster_name)
         max_retries = 10 if wait_master_fe_query_addr_file else 0
         i = 0
@@ -464,6 +463,7 @@ def compose(self):
             "volumes": volumes,
         }
 
+        extra_hosts = []
         if self.cluster.is_host_network():
             content["network_mode"] = "host"
         else:
@@ -473,11 +473,16 @@ def compose(self):
                     "ipv4_address": self.get_ip(),
                 }
             }
-            content["extra_hosts"] = [
+            extra_hosts.extend([
                 "{}:{}".format(node.get_name(), node.get_ip())
                 for node in self.cluster.get_all_nodes()
-            ]
+            ])
             content["ports"] = self.docker_ports()
+        user_hosts = getattr(self.cluster, "extra_hosts", [])
+        if user_hosts:
+            extra_hosts.extend(user_hosts)
+        if extra_hosts:
+            content["extra_hosts"] = extra_hosts
 
         if self.entrypoint():
             content["entrypoint"] = self.entrypoint()
@@ -780,8 +785,8 @@ class Cluster(object):
     def __init__(self, name, subnet, image, is_cloud, is_root_user, fe_config,
                  be_config, ms_config, recycle_config, remote_master_fe,
                  local_network_ip, fe_follower, be_disks, be_cluster, reg_be,
-                 coverage_dir, cloud_store_config, sql_mode_node_mgr,
-                 be_metaservice_endpoint, be_cluster_id):
+                 extra_hosts, coverage_dir, cloud_store_config,
+                 sql_mode_node_mgr, be_metaservice_endpoint, be_cluster_id):
         self.name = name
         self.subnet = subnet
         self.image = image
@@ -797,6 +802,7 @@ def __init__(self, name, subnet, image, is_cloud, is_root_user, fe_config,
         self.be_disks = be_disks
         self.be_cluster = be_cluster
         self.reg_be = reg_be
+        self.extra_hosts = extra_hosts
         self.coverage_dir = coverage_dir
         self.cloud_store_config = cloud_store_config
         self.groups = {
@@ -813,9 +819,9 @@ def __init__(self, name, subnet, image, is_cloud, is_root_user, fe_config,
     @staticmethod
     def new(name, image, is_cloud, is_root_user, fe_config, be_config,
             ms_config, recycle_config, remote_master_fe, local_network_ip,
-            fe_follower, be_disks, be_cluster, reg_be, coverage_dir,
-            cloud_store_config, sql_mode_node_mgr, be_metaservice_endpoint,
-            be_cluster_id):
+            fe_follower, be_disks, be_cluster, reg_be, extra_hosts,
+            coverage_dir, cloud_store_config, sql_mode_node_mgr,
+            be_metaservice_endpoint, be_cluster_id):
         if not os.path.exists(LOCAL_DORIS_PATH):
             os.makedirs(LOCAL_DORIS_PATH, exist_ok=True)
             os.chmod(LOCAL_DORIS_PATH, 0o777)
@@ -827,9 +833,10 @@ def new(name, image, is_cloud, is_root_user, fe_config, be_config,
             cluster = Cluster(name, subnet, image, is_cloud, is_root_user,
                               fe_config, be_config, ms_config, recycle_config,
                               remote_master_fe, local_network_ip, fe_follower,
-                              be_disks, be_cluster, reg_be, coverage_dir,
-                              cloud_store_config, sql_mode_node_mgr,
-                              be_metaservice_endpoint, be_cluster_id)
+                              be_disks, be_cluster, reg_be, extra_hosts,
+                              coverage_dir, cloud_store_config,
+                              sql_mode_node_mgr, be_metaservice_endpoint,
+                              be_cluster_id)
             os.makedirs(cluster.get_path(), exist_ok=True)
             os.makedirs(get_status_path(name), exist_ok=True)
             cluster._save_meta()
@@ -860,7 +867,7 @@ def _get_meta_file(name):
         return os.path.join(get_cluster_path(name), "meta")
 
     def is_host_network(self):
-        return getattr(self, "remote_master_fe", "")
+        return self.remote_master_fe
 
     def get_remote_fe_node(self):
         if not self.is_host_network():

docker/runtime/doris-compose/command.py

@@ -430,6 +430,14 @@ def add_parser(self, args_parsers):
                            help="Recreate containers even if their configuration " \
                                 "and image haven't changed. ")
 
+        parser.add_argument(
+            "--extra-hosts",
+            nargs="*",
+            type=str,
+            help=
+            "Add custom host-to-IP mappings (host:ip). For example: --extra-hosts myhost1:192.168.10.1 myhost2:192.168.10.2 . Only use when creating new cluster."
+        )
+
         parser.add_argument("--coverage-dir",
                             default="",
                             help="Set code coverage output directory")
@@ -599,8 +607,8 @@ def run(self, args):
                 args.NAME, args.IMAGE, args.cloud, args.root, args.fe_config,
                 args.be_config, args.ms_config, args.recycle_config,
                 args.remote_master_fe, args.local_network_ip, args.fe_follower,
-                args.be_disks, args.be_cluster, args.reg_be, args.coverage_dir,
-                cloud_store_config, args.sql_mode_node_mgr,
+                args.be_disks, args.be_cluster, args.reg_be, args.extra_hosts,
+                args.coverage_dir, cloud_store_config, args.sql_mode_node_mgr,
                 args.be_metaservice_endpoint, args.be_cluster_id)
             LOG.info("Create new cluster {} succ, cluster path is {}".format(
                 args.NAME, cluster.get_path()))

regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy

@@ -50,6 +50,7 @@ class ClusterOptions {
     // for example, ' xx = yy ' is bad, should use 'xx=yy'
     List<String> feConfigs = [
         'heartbeat_interval_second=5',
+        'workload_group_check_interval_ms=1000',
     ]
 
     // don't add whitespace in beConfigs items,
@@ -64,6 +65,10 @@ class ClusterOptions {
 
     List<String> recycleConfigs = []
 
+    // host mapping(host:IP), for example: myhost:192.168.10.10
+    // just as `docker run --add-host myhost:192.168.10.10` do.
+    List<String> extraHosts = []
+
     boolean connectToFollower = false
 
     // 1. cloudMode = true, only create cloud cluster.
@@ -306,26 +311,30 @@ class SuiteCluster {
             cmd += ['--add-ms-num', String.valueOf(options.msNum)]
         }
         // TODO: need escape white space in config
-        if (options.feConfigs != null && options.feConfigs.size() > 0) {
+        if (!options.feConfigs.isEmpty()) {
             cmd += ['--fe-config']
             cmd += options.feConfigs
         }
-        if (options.beConfigs != null && options.beConfigs.size() > 0) {
+        if (!options.beConfigs.isEmpty()) {
             cmd += ['--be-config']
             cmd += options.beConfigs
         }
-        if (options.msConfigs != null && options.msConfigs.size() > 0) {
+        if (!options.msConfigs.isEmpty()) {
             cmd += ['--ms-config']
             cmd += options.msConfigs
         }
-        if (options.recycleConfigs != null && options.recycleConfigs.size() > 0) {
+        if (!options.recycleConfigs.isEmpty()) {
             cmd += ['--recycle-config']
             cmd += options.recycleConfigs
         }
         if (options.beDisks != null) {
             cmd += ['--be-disks']
             cmd += options.beDisks
         }
+        if (!options.extraHosts.isEmpty()) {
+            cmd += ['--extra-hosts']
+            cmd += options.extraHosts
+        }
         if (config.dockerCoverageOutputDir != null && config.dockerCoverageOutputDir != '') {
             cmd += ['--coverage-dir', config.dockerCoverageOutputDir]
         }