Merge pull request #9676 from everpcpc/feat-minio

ci: support setup minio storage & external s3 storage in docker image

.github/workflows/release.yml

@@ -358,6 +358,14 @@ jobs:
           context: .
           file: ./docker/Dockerfile
           build-args: VERSION=${{ needs.create_release.outputs.version }}
+      - name: Update repo description
+        uses: peter-evans/dockerhub-description@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+          repository: ${{ steps.login.outputs.dockerhub_repo }}
+          short-description: ${{ github.event.repository.description }}
+          readme-filepath: ./docker/README.md
 
   release_docker_separate:
     name: docker image seperated

docker/Dockerfile

@@ -8,8 +8,10 @@ RUN apt-get update -y && \
     rm -rf /var/lib/apt/lists/* && \
     rm -rf /var/cache/apt/*
 
-COPY ./distro/$TARGETPLATFORM/databend-meta /databend-meta
-COPY ./distro/$TARGETPLATFORM/databend-query /databend-query
-COPY ./docker/databend-query-docker.toml /databend-query.toml
+COPY ./distro/$TARGETPLATFORM/databend-meta /usr/bin/databend-meta
+COPY ./distro/$TARGETPLATFORM/databend-query /usr/bin/databend-query
+COPY ./docker/query-config.toml /etc/databend/query.toml
 COPY ./docker/bootstrap.sh /bootstrap.sh
 ENTRYPOINT [ "dumb-init", "--", "/bootstrap.sh"]
+VOLUME [ "/var/log/databend", "/etc/databend", "/var/lib/databend", "/var/lib/minio" ]
+EXPOSE 3307 8124 8000

docker/README.md

@@ -1,50 +1,78 @@
-# Multi-architecture Docker image support
+# Databend All-in-One Docker Image
 
-Support cross platform docker image build
+Support Platform: `linux/amd64`, `linux/arm64`
 
-## How to use
 
-## Install docker ![Buildkit](https://github.com/moby/buildkit)
+## Available Enviroment Variables
 
-For macOS
+* DATABEND_QUERY_CONFIG_FILE
+* DATABEND_QUERY_DEFAULT_USER
+* DATABEND_QUERY_DEFAULT_PASSWORD
+* DATABEND_QUERY_STORAGE_TYPE
 
-```bash
-brew install buildkit
-```
+* AWS_S3_ENDPOINT
+* AWS_S3_PRESIGNED_ENDPOINT
+* AWS_S3_BUCKET
+* AWS_ACCESS_KEY_ID
+* AWS_SECRET_ACCESS_KEY
 
-For linux
+* MINIO_ENABLED
 
-```bash
-docker run --name buildkit -d --privileged -p 1234:1234 moby/buildkit --addr tcp://0.0.0.0:1234
-export BUILDKIT_HOST=tcp://0.0.0.0:1234
-docker cp buildkit:/usr/bin/buildctl /usr/local/bin/
-buildctl build --help
-```
 
-## Check on available platforms given your host machine
+## How to use
+
 
-```bash
-docker run --privileged --rm tonistiigi/binfmt --install all
+### Run default config with fs backend
+```
+docker run -p 8000:8000 datafuselabs/databend
 ```
 
-## Build and push container for supported platforms
-
-### initialize host networking
+### Run with MinIO as backend
+```
+docker run \
+    -p 8000:8000 \
+    -p 9000:9000 \
+    -e MINIO_ENABLED=true \
+    datafuselabs/databend
+```
 
-```bash
-docker buildx create --name host --use --buildkitd-flags '--allow-insecure-entitlement network.host'
+### Adding built-in query user
+```
+docker run \
+    -p 8000:8000 \
+    -e DATABEND_QUERY_DEFAULT_USER=databend \
+    -e DATABEND_QUERY_DEFAULT_PASSWORD=databend \
+    datafuselabs/databend
 ```
 
-### update qemu static link
+### Run with external S3 service
 
-```bash
- docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+```
+docker run \
+    -p 8000:8000 \
+    -e DATABEND_QUERY_STORAGE_TYPE=s3 \
+    -e AWS_S3_ENDPOINT="http://some_s3_endpoint \
+    -e AWS_S3_BUCKET=some_bucket \
+    -e AWS_ACCESS_KEY_ID=some_key \
+    -e AWS_SECRET_ACCESS_KEY=some_secret \
+    datafuselabs/databend
 ```
 
-### build with given buildx builder
-
-return to databend root directory
+### Run with persistent local storage & logs
+```
+docker run \
+    -p 8000:8000 \
+    -v meta_storage_dir:/var/lib/databend/meta \
+    -v query_storage_dir:/var/lib/databend/query \
+    -v log_dir:/var/log/databend \
+    datafuselabs/databend
+```
 
-```bash
-make dockerx PLATFORM=<platform your host machine supports>
-```
+### Run with self managed query config
+```
+docker run \
+    -p 8000:8000 \
+    -e DATABEND_QUERY_CONFIG_FILE=/etc/databend/mine.toml \
+    -v query_config_file:/etc/databend/mine.toml \
+    datafuselabs/databend
+```

docker/bootstrap.sh

@@ -1,30 +1,114 @@
 #!/bin/bash
 
-# DATABEND_DEFAULT_USER="${DATABEND_DEFAULT_USER:-databend}"
-# DATABEND_DEFAULT_PASSWORD="${DATABEND_DEFAULT_PASSWORD:-databend}"
+PROCESSES=()
 
-/databend-meta --single &>/tmp/databend-meta.log &
-P1=$!
+mkdir -p /var/log/databend
+mkdir -p /var/lib/databend/meta
+mkdir -p /var/lib/databend/query
 
-# TODO: add health check to remove the race condition issue during databend-query bootstrap
+databend-meta \
+    --log-file-dir /var/log/databend \
+    --log-stderr-level WARN \
+    --raft-dir /var/lib/databend/meta \
+    --single &>/tmp/std-meta.log &
+PROCESSES+=($!)
+# wait for meta to be ready
 sleep 1
 
-if [ -n "$DATABEND_DEFAULT_USER" ] && [ -n "$DATABEND_DEFAULT_PASSWORD" ]; then
-    DOUBLE_SHA1_PASSWORD=$(echo -n "$DATABEND_DEFAULT_PASSWORD" | sha1sum | cut -d' ' -f1 | xxd -r -p | sha1sum | cut -d' ' -f1)
-    cat <<EOF >>databend-query.toml
+function setup_query_default_user {
+    if [ -n "$DATABEND_QUERY_DEFAULT_USER" ] && [ -n "$DATABEND_QUERY_DEFAULT_PASSWORD" ]; then
+        DOUBLE_SHA1_PASSWORD=$(echo -n "$DATABEND_QUERY_DEFAULT_PASSWORD" | sha1sum | cut -d' ' -f1 | xxd -r -p | sha1sum | cut -d' ' -f1)
+        cat <<EOF >>"$QUERY_CONFIG_FILE"
 [[query.users]]
-name = "$DATABEND_DEFAULT_USER"
+name = "$DATABEND_QUERY_DEFAULT_USER"
 auth_type = "double_sha1_password"
 auth_string = "$DOUBLE_SHA1_PASSWORD"
 EOF
+    fi
+}
+
+function setup_minio {
+    echo "==> Downloading MinIO server..."
+    arch=$(uname -m)
+    case $arch in
+    "x86_64")
+        platform="amd64"
+        ;;
+    "aarch64")
+        platform="arm64"
+        ;;
+    *)
+        echo "==> Unsupported architecture: $arch"
+        exit 1
+        ;;
+    esac
+    curl -sSLo /usr/bin/minio "https://dl.min.io/server/minio/release/linux-${platform}/minio"
+    echo "==> MinIO server downloaded."
+    chmod +x /usr/bin/minio
+    mkdir -p /var/lib/minio
+    # make sure the bucket exists
+    mkdir -p "/var/lib/minio/$AWS_S3_BUCKET"
+    MINIO_ROOT_USER="$AWS_ACCESS_KEY_ID" MINIO_ROOT_PASSWORD="$AWS_SECRET_ACCESS_KEY" minio server /var/lib/minio &>/tmp/std-minio.log &
+    PROCESSES+=($!)
+    # wait for minio to be ready
+    sleep 1
+}
+
+function setup_query_storage {
+    QUERY_STORAGE_TYPE=${QUERY_STORAGE_TYPE:-"fs"}
+    if [[ -n $MINIO_ENABLED ]]; then
+        # force to use s3 storage when minio is enabled
+        QUERY_STORAGE_TYPE="s3"
+        AWS_S3_BUCKET=${AWS_S3_BUCKET:-"databend"}
+        AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT:-"http://0.0.0.0:9000"}
+        AWS_S3_PRESIGNED_ENDPOINT=${AWS_S3_PRESIGNED_ENDPOINT:-"http://localhost:9000"}
+        AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-"minio"}
+        AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-"miniostorage"}
+        setup_minio
+    fi
+    case $QUERY_STORAGE_TYPE in
+    "fs")
+        mkdir -p /var/lib/databend/query
+        cat <<EOF >>"$QUERY_CONFIG_FILE"
+[storage]
+type = "fs"
+[storage.fs]
+data_path = "/var/lib/databend/query"
+EOF
+        ;;
+    "s3")
+        cat <<EOF >>"$QUERY_CONFIG_FILE"
+[storage]
+type = "s3"
+[storage.s3]
+bucket = "$AWS_S3_BUCKET"
+endpoint_url = "$AWS_S3_ENDPOINT"
+access_key_id = "$AWS_ACCESS_KEY_ID"
+secret_access_key = "$AWS_SECRET_ACCESS_KEY"
+presign_endpoint_url = "$AWS_S3_PRESIGNED_ENDPOINT"
+EOF
+        ;;
+
+    *)
+        echo "==> Unsupported storage type: $QUERY_STORAGE_TYPE"
+        exit 1
+        ;;
+    esac
+}
+
+if [ -z "$QUERY_CONFIG_FILE" ]; then
+    QUERY_CONFIG_FILE="/etc/databend/query.toml"
+    echo "==> QUERY_CONFIG_FILE is not set, using default: $QUERY_CONFIG_FILE"
+    setup_query_default_user
+    setup_query_storage
 fi
 
-/databend-query -c databend-query.toml &>/tmp/databend-query.log &
-P2=$!
+databend-query -c "$QUERY_CONFIG_FILE" &>/tmp/std-query.log &
+PROCESSES+=($!)
+# wait for query to be ready
+sleep 1
 
-tail -f /tmp/databend-query.log &
-P3=$!
+tail -n +1 -F /tmp/std-*.log &
+PROCESSES+=($!)
 
-tail -f /tmp/databend-meta.log &
-P4=$!
-wait $P1 $P2 $P3 $P4
+wait "${PROCESSES[@]}"

docker/databend-query-docker.toml → docker/query-config.toml

@@ -1,29 +1,17 @@
-# Usage:
-# databend-query -c databend_query_config_spec.toml
-
 [query]
 max_active_sessions = 256
 wait_timeout_mills = 5000
 
-# For flight rpc.
 flight_api_address = "0.0.0.0:9090"
-
-# Databend Query http address.
-# For admin RESET API.
 admin_api_address = "0.0.0.0:8080"
-
-# Databend Query metrics RESET API.
 metric_api_address = "0.0.0.0:7070"
 
-# Databend Query MySQL Handler.
 mysql_handler_host = "0.0.0.0"
 mysql_handler_port = 3307
 
-# Databend Query ClickHouse HTTP Handler.
 clickhouse_http_handler_host = "0.0.0.0"
 clickhouse_http_handler_port = 8124
 
-# Databend Query HTTP Handler.
 http_handler_host = "0.0.0.0"
 http_handler_port = 8000
 
@@ -32,23 +20,16 @@ cluster_id = "default"
 
 [log]
 
-[log.file]
-level = "INFO"
+[log.stderr]
+level = "WARN"
 format = "text"
-dir = "./.databend/logs"
 
+[log.file]
+level = "INFO"
+dir = "/var/log/databend"
 
 [meta]
 endpoints = ["0.0.0.0:9191"]
 username = "root"
 password = "root"
 client_timeout_in_second = 60
-
-# Storage config.
-[storage]
-# fs|s3
-type = "fs"
-
-# fs storage.
-[storage.fs]
-data_path = "./.databend/data"