Skip to content

Commit

Permalink
Merge branch 'dev' into issues/16481
Browse files Browse the repository at this point in the history
  • Loading branch information
SbloodyS authored Sep 5, 2024
2 parents e2dca9a + fee4955 commit bdca410
Show file tree
Hide file tree
Showing 431 changed files with 3,939 additions and 5,249 deletions.
27 changes: 18 additions & 9 deletions .github/workflows/api-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -89,26 +89,35 @@ jobs:
strategy:
matrix:
case:
- name: Tenant
- name: TenantAPITest
class: org.apache.dolphinscheduler.api.test.cases.TenantAPITest
- name: WorkerGroup
- name: WorkerGroupAPITest
class: org.apache.dolphinscheduler.api.test.cases.WorkerGroupAPITest
- name: Project
- name: ProjectAPITest
class: org.apache.dolphinscheduler.api.test.cases.ProjectAPITest
- name: Workflow
class: org.apache.dolphinscheduler.api.test.cases.ProcessDefinitionAPITest
- name: Scheduler
- name: WorkflowDefinitionAPITest
class: org.apache.dolphinscheduler.api.test.cases.WorkflowDefinitionAPITest
- name: SchedulerAPITest
class: org.apache.dolphinscheduler.api.test.cases.SchedulerAPITest
- name: Executor
- name: ExecutorAPITest
class: org.apache.dolphinscheduler.api.test.cases.ExecutorAPITest
- name: ProcessInstance
class: org.apache.dolphinscheduler.api.test.cases.ProcessInstanceAPITest
- name: WorkflowInstanceAPITest
class: org.apache.dolphinscheduler.api.test.cases.WorkflowInstanceAPITest
env:
RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
steps:
- uses: actions/checkout@v4
with:
submodules: true
- name: Set up JDK 11
uses: actions/setup-java@v4
with:
java-version: 11
distribution: 'adopt'
- name: Collect Workflow Telemetry
uses: ./.github/actions/workflow-telemetry-action
with:
comment_on_pr: false
- name: Cache local Maven repository
uses: actions/cache@v4
with:
Expand Down
137 changes: 18 additions & 119 deletions .github/workflows/backend.yml
Original file line number Diff line number Diff line change
Expand Up @@ -132,136 +132,35 @@ jobs:
run: |
/bin/bash ${{ matrix.case.script }}
schema-check:
name: ${{ matrix.case.name }}-${{ matrix.version }}
needs: build
runs-on: ubuntu-latest
if: ${{ (needs.paths-filter.outputs.db-schema == 'true') || (github.event_name == 'push') }}
timeout-minutes: 20
needs: build
services:
mysql:
image: mysql:5.7
env:
MYSQL_ROOT_PASSWORD: mysql
MYSQL_DATABASE: dolphinscheduler_dev
ports:
- 3306:3306
options: --name=mysql --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
postgres:
image: postgres:15
env:
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dolphinscheduler_dev
ports:
- 5432:5432
options: --name=postgres --health-cmd=pg_isready --health-interval=10s --health-timeout=5s --health-retries=5
strategy:
fail-fast: false
matrix:
db: ["mysql", "postgresql"]
version: ["3.1.9", "3.2.0"]
case:
- name: schema-check-with-mysql
script: .github/workflows/schema-check/mysql/start-job.sh
- name: schema-check-with-postgresql
script: .github/workflows/schema-check/postgresql/start-job.sh
steps:
- name: Set up JDK 8
uses: actions/setup-java@v4
- uses: actions/checkout@v4
with:
java-version: 8
distribution: 'adopt'
- name: Install Atlas and Create Dir
run: |
mkdir -p dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}
curl -sSf https://atlasgo.sh | sh
- name: Download Tarball
submodules: true
- name: Collect Workflow Telemetry
uses: ./.github/actions/workflow-telemetry-action
with:
comment_on_pr: false
- name: Download Binary Package
uses: actions/download-artifact@v4
with:
name: binary-package-8
path: dolphinscheduler/dev
- name: Set Env
run: |
VERSION=${{ matrix.version }}
echo "DATABASE_VERSION=${VERSION//\./}" >> $GITHUB_ENV
- name: Prepare
run: |
wget https://archive.apache.org/dist/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P dolphinscheduler/${{ matrix.version }}
tar -xzf dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -C dolphinscheduler/${{ matrix.version }} --strip-components 1
tar -xzf dolphinscheduler/dev/apache-dolphinscheduler-*-bin.tar.gz -C dolphinscheduler/dev --strip-components 1
if [[ ${{ matrix.db }} == "mysql" ]]; then
MYSQL_JDBC_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
MYSQL_JDBC_JAR="mysql-connector-java-8.0.16.jar"
wget ${MYSQL_JDBC_URL} -O /tmp/${MYSQL_JDBC_JAR}
for base_dir in dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}; do
if [[ $base_dir == *"dolphinscheduler/2"* ]]; then
cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/lib
else
for d in alert-server api-server master-server worker-server tools; do
cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/${d}/libs
done
fi
done
docker exec -i mysql mysql -uroot -pmysql -e "create database dolphinscheduler_${{ env.DATABASE_VERSION }}";
else
docker exec -i postgres psql -U postgres -c "create database dolphinscheduler_${{ env.DATABASE_VERSION }};"
fi
- name: Check
path: ds_schema_check_test/dev
- name: Running Schema Check
run: |
if [[ $DATABASE_VERSION -lt 300 ]]; then
chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
else
chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
fi
if [[ ${{ matrix.db }} == "mysql" ]]; then
export DATABASE="mysql"
export SPRING_DATASOURCE_DRIVER_CLASS_NAME="com.mysql.cj.jdbc.Driver"
export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_dev?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
export SPRING_DATASOURCE_USERNAME="root"
export SPRING_DATASOURCE_PASSWORD="mysql"
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
if [[ $DATABASE_VERSION -lt 300 ]]; then
bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
else
bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
fi
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
atlas_result=$(atlas schema diff \
--from "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}" \
--to "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_dev")
if [[ ${atlas_result} != *"Schemas are synced"* ]]; then
echo "================================================================================================"
echo " !!!!! For Contributors !!!!!"
echo "================================================================================================"
echo "Database schema not sync, please add below change in the latest version of dolphinscheduler-dao/src/main/resources/sql/upgrade directory"
echo "${atlas_result}"
exit 1
fi
else
export DATABASE="postgresql"
export SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver"
export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_dev"
export SPRING_DATASOURCE_USERNAME="postgres"
export SPRING_DATASOURCE_PASSWORD="postgres"
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}"
if [[ $DATABASE_VERSION -lt 300 ]]; then
bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
else
bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
fi
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
atlas_result=$(atlas schema diff \
--from "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}?search_path=public&sslmode=disable" \
--to "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_dev?search_path=public&sslmode=disable")
if [[ ${atlas_result} != *"Schemas are synced"* ]]; then
echo "================================================================================================"
echo " !!!!! For Contributors !!!!!"
echo "================================================================================================"
echo "Database schema not sync, please add below change in the latest version in dolphinscheduler-dao/src/main/resources/sql/upgrade directory"
echo "${atlas_result}"
exit 1
fi
fi
/bin/bash ${{ matrix.case.script }} ${{ matrix.version }}
result:
name: Build
runs-on: ubuntu-latest
Expand All @@ -275,7 +174,7 @@ jobs:
echo "Skip Build!"
exit 0
fi
if [[ ${{ needs.build.result }} != 'success' || ${{ needs.cluster-test.result }} != 'success' ]]; then
if [[ ${{ needs.build.result }} != 'success' || ${{ needs.cluster-test.result }} != 'success' || ${{ needs.schema-check.result }} != 'success' ]]; then
echo "Build Failed!"
exit -1
fi
34 changes: 17 additions & 17 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,45 +92,45 @@ jobs:
strategy:
matrix:
case:
- name: Tenant
- name: TenantE2ETest
class: org.apache.dolphinscheduler.e2e.cases.TenantE2ETest
- name: User
- name: UserE2ETest
class: org.apache.dolphinscheduler.e2e.cases.UserE2ETest
- name: WorkerGroup
- name: WorkerGroupE2ETest
class: org.apache.dolphinscheduler.e2e.cases.WorkerGroupE2ETest
- name: Project
- name: ProjectE2ETest
class: org.apache.dolphinscheduler.e2e.cases.ProjectE2ETest
- name: Queue
- name: QueueE2ETest
class: org.apache.dolphinscheduler.e2e.cases.QueueE2ETest
- name: Environment
- name: EnvironmentE2ETest
class: org.apache.dolphinscheduler.e2e.cases.EnvironmentE2ETest
- name: Cluster
- name: ClusterE2ETest
class: org.apache.dolphinscheduler.e2e.cases.ClusterE2ETest
- name: Token
- name: TokenE2ETest
class: org.apache.dolphinscheduler.e2e.cases.TokenE2ETest
- name: Workflow
- name: WorkflowE2ETest
class: org.apache.dolphinscheduler.e2e.cases.WorkflowE2ETest
- name: WorkflowHttp
- name: WorkflowHttpTaskE2ETest
class: org.apache.dolphinscheduler.e2e.cases.WorkflowHttpTaskE2ETest
- name: WorkflowJava
- name: WorkflowJavaTaskE2ETest
class: org.apache.dolphinscheduler.e2e.cases.WorkflowJavaTaskE2ETest
# - name: WorkflowForSwitch
# class: org.apache.dolphinscheduler.e2e.cases.WorkflowSwitchE2ETest
- name: FileManage
- name: FileManageE2ETest
class: org.apache.dolphinscheduler.e2e.cases.FileManageE2ETest
- name: MysqlDataSource
- name: MysqlDataSourceE2ETest
class: org.apache.dolphinscheduler.e2e.cases.MysqlDataSourceE2ETest
- name: ClickhouseDataSource
- name: ClickhouseDataSourceE2ETest
class: org.apache.dolphinscheduler.e2e.cases.ClickhouseDataSourceE2ETest
- name: PostgresDataSource
- name: PostgresDataSourceE2ETest
class: org.apache.dolphinscheduler.e2e.cases.PostgresDataSourceE2ETest
- name: ShellTaskE2ETest
class: org.apache.dolphinscheduler.e2e.cases.tasks.ShellTaskE2ETest
- name: PythonTaskE2ETest
class: org.apache.dolphinscheduler.e2e.cases.tasks.PythonTaskE2ETest
- name: SqlServerDataSource
- name: SqlServerDataSourceE2ETest
class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest
- name: HiveDataSource
- name: HiveDataSourceE2ETest
class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest
env:
RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
Expand Down
35 changes: 35 additions & 0 deletions .github/workflows/schema-check/mysql/docker-compose-base.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

version: "3"

services:
mysql:
container_name: mysql
image: mysql:8.0.33
command: --default-authentication-plugin=mysql_native_password
restart: always
environment:
MYSQL_ROOT_PASSWORD: mysql
MYSQL_DATABASE: dolphinscheduler_dev
ports:
- "3306:3306"
healthcheck:
test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD
interval: 5s
timeout: 60s
retries: 120
54 changes: 54 additions & 0 deletions .github/workflows/schema-check/mysql/running-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -euox pipefail

DS_VERSION=$1
DATABASE_VERSION=$2

# Install dev schema
export DATABASE="mysql"
export SPRING_DATASOURCE_DRIVER_CLASS_NAME="com.mysql.cj.jdbc.Driver"
export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_dev?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
export SPRING_DATASOURCE_USERNAME="root"
export SPRING_DATASOURCE_PASSWORD="mysql"
bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh

# Install the target version schema and upgrade it
export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_${DATABASE_VERSION}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
bash ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh
bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh

# Compare the schema
set +x
atlas_result=$(atlas schema diff \
--from "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_${DATABASE_VERSION}" \
--to "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_dev")
if [[ ${atlas_result} != *"Schemas are synced"* ]]; then
echo "================================================================================================"
echo " !!!!! For Contributors !!!!!"
echo "================================================================================================"
echo "Database schema not sync, please add below change in the latest version of dolphinscheduler-dao/src/main/resources/sql/upgrade directory"
echo "${atlas_result}"
exit 1
else
echo "================================================================================================"
echo " !!!!! For Contributors !!!!!"
echo "================================================================================================"
echo "Database schema sync successfully"
exit 0
fi
Loading

0 comments on commit bdca410

Please sign in to comment.