Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add cicd pipeline #16

Merged
merged 2 commits into from
Jul 19, 2023
Merged

Add cicd pipeline #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
01bf816
add cicd pipeline
sandiandian Jun 13, 2023
574918d
Fix: remove qingcloud platform-specific unit-tests
sandiandian Jul 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 88 additions & 0 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
name: cbdb pipeline
on:
pull_request:
branches:
- main

env:
BUILD_TYPE: "debug"
FTS_MODE: "internal_fts"
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}

permissions:
id-token: write
contents: read
jobs:
build:
runs-on: [self-hosted, example]
steps:
- uses: actions/checkout@v3
- name: Run build script
run: |
echo $GITHUB_RUN_ID > BUILD_NUMBER
hd-ci/compile_cbdb.bash ${GEN_DOCKER_TAG}
- uses: actions/upload-artifact@v3
with:
name: cbdb-variables
path: /opt/actions-runner/_work/cloudberrydb/cloudberrydb/cbdb-artifacts.txt
- uses: actions/upload-artifact@v3
with:
name: cbdb-package
path: /tmp/server-build*
icw-test:
needs: build
runs-on: [self-hosted, example]
env:
MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=off' installcheck-world"
TEST_OS: "centos"
DUMP_DB: "true"
steps:
- uses: actions/checkout@v3
with:
path: "gpdb_src"
- uses: actions/download-artifact@v3
with:
name: cbdb-variables
path: /opt/
- uses: actions/download-artifact@v3
with:
name: cbdb-package
path: /opt/
- name: Run icw-test script
run: |
mkdir /code
cp -a gpdb_src/ /code
cd /code
echo $GITHUB_RUN_ID > gpdb_src/BUILD_NUMBER
gpdb_src/hd-ci/icw_cbdb.bash $FTS_MODE
icw-orca-test:
needs: build
runs-on: [self-hosted, example]
env:
MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=on' installcheck-world"
TEST_OS: "centos"
DUMP_DB: "true"
steps:
- uses: actions/checkout@v3
with:
path: "gpdb_src"
- uses: actions/download-artifact@v3
with:
name: cbdb-variables
path: /opt/
- uses: actions/download-artifact@v3
with:
name: cbdb-package
path: /opt/
- name: Run icw-orca-test script
run: |
mkdir /code
cp -a gpdb_src/ /code
cd /code
echo $GITHUB_RUN_ID > gpdb_src/BUILD_NUMBER
gpdb_src/hd-ci/icw_cbdb.bash $FTS_MODE




137 changes: 137 additions & 0 deletions .github/workflows/release.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
name: cbdb release
on:
push:
tags:
- 'v*'

env:
BUILD_TYPE: "release"
FTS_MODE: "internal_fts"
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
GEN_DOCKER_TAG: true
INSTALL_DIR: "/usr/local/cloudberry-db"
CLIENT_INSTALL_DIR: "/usr/local/cloudberry-clients"


jobs:
build:
runs-on: [self-hosted, default]
steps:
- uses: actions/checkout@v3
- name: Run build script
run: |
echo $GITHUB_RUN_ID > BUILD_NUMBER
hd-ci/compile_cbdb.bash $GEN_DOCKER_TAG
- uses: actions/upload-artifact@v3
with:
name: cbdb-variables
path: /opt/actions-runner/_work/cloudberrydb/cloudberrydb/cbdb-artifacts.txt
- uses: actions/upload-artifact@v3
with:
name: cbdb-package
path: /tmp/server-build*
- uses: actions/upload-artifact@v3
with:
name: cbdb-rpm
path: /tmp/cloudberry-db*.rpm
icw-test:
needs: build
runs-on: [self-hosted, example]
env:
MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=off' installcheck-world"
TEST_OS: "centos"
DUMP_DB: "true"
BUILD_TYPE: release
steps:
- uses: actions/checkout@v3
with:
path: "gpdb_src"
- uses: actions/download-artifact@v3
with:
name: cbdb-variables
path: /opt/
- uses: actions/download-artifact@v3
with:
name: cbdb-package
path: /opt/
- uses: actions/download-artifact@v3
with:
name: cbdb-rpm
path: /opt/
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v2
with:
aws-region: us-east-1
- name: Run icw-test script
run: |
mkdir /code
cp -a gpdb_src/ /code
cd /code
echo $GITHUB_RUN_ID > gpdb_src/BUILD_NUMBER
gpdb_src/hd-ci/icw_cbdb.bash internal_fts
icw-orca-test:
needs: build
runs-on: [self-hosted, example]
env:
MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=on' installcheck-world"
TEST_OS: "centos"
DUMP_DB: "true"
BUILD_TYPE: release
steps:
- uses: actions/checkout@v3
with:
path: "gpdb_src"
- uses: actions/download-artifact@v3
with:
name: cbdb-variables
path: /opt/
- uses: actions/download-artifact@v3
with:
name: cbdb-package
path: /opt/
- uses: actions/download-artifact@v3
with:
name: cbdb-rpm
path: /opt/
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v2
with:
aws-region: us-east-1
- name: Run icw-orca-test script
run: |
mkdir /code
cp -a gpdb_src/ /code
cd /code
echo $GITHUB_RUN_ID > gpdb_src/BUILD_NUMBER
gpdb_src/hd-ci/icw_cbdb.bash internal_fts
release:
name: Upload Release Asset
needs: ["icw-orca-test","icw-test"]
runs-on: ubuntu-latest
steps:
- name: Create Release
id: create_release
uses: actions/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ github.ref }}
release_name: Release ${{ github.ref }}
draft: false
prerelease: false
- uses: actions/download-artifact@v3
with:
name: cbdb-package
path: /opt/
- uses: actions/download-artifact@v3
with:
name: cbdb-rpm
path: /opt/
- name: Upload Release Asset
id: upload-release-asset
uses: alexellis/upload-assets@master
env:
GITHUB_TOKEN: ${{ github.token }}
with:
asset_paths: '["/opt/server-build*","/opt/cloudberry-db-*"]'
28 changes: 11 additions & 17 deletions concourse/scripts/common.bash
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,11 @@ function download_etcd() {
etcd_file_name=etcd-${etcd_version}-linux-arm64
fi

etcd_download_url=https://artifactory.hashdata.xyz/artifactory/utility/${etcd_file_name}.tar.gz

wget ${etcd_download_url} -O /tmp/${etcd_file_name}.tar.gz
tar -xvf /tmp/${etcd_file_name}.tar.gz -C /tmp
tar -xvf /opt/${etcd_file_name}.tar.gz -C /opt

mkdir -p ${target_path}
\cp /tmp/${etcd_file_name}/etcd ${target_path}
\cp /tmp/${etcd_file_name}/etcdctl ${target_path}
rm -rf /tmp/${etcd_file_name} /tmp/${etcd_file_name}.tar.gz
\cp /opt/${etcd_file_name}/etcd ${target_path}
\cp /opt/${etcd_file_name}/etcdctl ${target_path}
}

function download_jansson() {
Expand All @@ -59,16 +55,16 @@ function download_jansson() {
fi
jansson_file_name=jansson-${jansson_version}

wget https://artifactory.hashdata.xyz/artifactory/utility/${jansson_file_name}.tar.gz -O /tmp/${jansson_file_name}.tar.gz
tar -xvf /tmp/${jansson_file_name}.tar.gz -C /tmp
tar -xvf /opt/${jansson_file_name}.tar.gz -C /opt

pushd /tmp/${jansson_file_name}
./configure --prefix=/tmp/jansson --disable-static
pushd /opt/${jansson_file_name}
./configure --prefix=/opt/jansson --disable-static
make && make install

mkdir -p ${target_path}
\cp -p /tmp/jansson/lib/libjansson.so* ${target_path}
rm -rf /tmp/jansson /tmp/${jansson_file_name}.tar.gz /tmp/${jansson_file_name}
\cp -p /opt/jansson/lib/libjansson.so* ${target_path}
\cp -p /opt/jansson/include/jansson*.h /usr/include/
rm -rf /opt/jansson
popd
}

Expand All @@ -79,12 +75,10 @@ function download_java() {
echo "invalid java target path!" && exit 1
fi
java_file_name=jdk-${java_version}-linux-`arch`
wget http://artifactory.hashdata.xyz/artifactory/development/centos/7/`arch`/${java_file_name}.tar.gz -O /tmp/${java_file_name}.tar.gz

mkdir -p ${target_path}
tar -xzf /tmp/${java_file_name}.tar.gz -C ${target_path}
tar -xzf /opt/${java_file_name}.tar.gz -C ${target_path}
mv ${target_path}/jdk* ${target_path}/jdk
rm /tmp/${java_file_name}.tar.gz
}

## ----------------------------------------------------------------------
Expand All @@ -93,7 +87,7 @@ function download_java() {

function install_gpdb() {
mkdir -p $INSTALL_DIR
tar -xzf bin_gpdb/bin_gpdb.tar.gz -C $INSTALL_DIR
tar -xzf /opt/bin_gpdb.tar.gz -C $INSTALL_DIR
}

function setup_configure_vars() {
Expand Down
20 changes: 4 additions & 16 deletions deploy/k8s/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,31 +1,19 @@
FROM docker.artifactory.hashdata.xyz/docker/cbdb:centosbase_k8s_1.0
FROM cloudberrydb/cloudberrydb:centosbase_k8s_1.0
ARG PACKAGE
ARG BUILD_TYPE
LABEL description="dockerfile for cbdb k8s auto-deployment"
RUN curl -O http://artifactory.hashdata.xyz/artifactory/hashdata-repository/intermediate-artifacts/centos7/x86_64/cbdb/${BUILD_TYPE}/${PACKAGE} && \
yum -y install R apr apr-devel apr-util automake autoconf bash bison bison-devel bzip2 bzip2-devel centos-release-scl curl flex flex-devel gcc gcc-c++ git gdb ibxml2 iproute krb5 krb5-devel less libcurl libcurl-devel libevent libevent-devel libxml2 libxml2-devel libyaml libzstd-devel libzstd.x86_64 make openldap openssh-client openssl openssl-devel openssl-libs perl python3-devel readline readline-devel rsync sed sudo tar vim wget which xerces-c-devel zip zip-devel zlib && \
ADD ${PACKAGE} .
RUN yum -y install R apr apr-devel apr-util automake autoconf bash bison bison-devel bzip2 bzip2-devel centos-release-scl curl flex flex-devel gcc gcc-c++ git gdb ibxml2 iproute krb5 krb5-devel less libcurl libcurl-devel libevent libevent-devel libxml2 libxml2-devel libyaml libzstd-devel libzstd.x86_64 make openldap openssh-client openssl openssl-devel openssl-libs perl python3-devel readline readline-devel rsync sed sudo tar vim wget which xerces-c-devel zip zip-devel zlib && \
yum -y install epel-release.noarch && \
yum -y install libzstd.x86_64 && \
yum -y install devtoolset-9-gcc* && \
wget https://artifactory.hashdata.xyz/artifactory/utility/etcd-v3.3.25-linux-amd64.tar.gz && \
wget https://cbdb-deps.s3.amazonaws.com/etcd-v3.3.25-linux-amd64.tar.gz && \
tar zxvf etcd-v3.3.25-linux-amd64.tar.gz && \
mv etcd-v3.3.25-linux-amd64/etcd /usr/local/bin/etcd && \
mv etcd-v3.3.25-linux-amd64/etcdctl /usr/local/bin/etcdctl && \
rm -rf etcd-v3.3.25-linux-amd64.tar.gz && \
rpm -ivh ${PACKAGE} && \
rm ${PACKAGE} && \
wget https://artifactory.hashdata.xyz/artifactory/utility/jansson-2.13.1.tar.gz && \
tar -xvf jansson-2.13.1.tar.gz && \
rm -rf jansson-2.13.1.tar.gz && \
cd jansson-2.13.1 && \
./configure --prefix=/usr --disable-static && \
make && \
make install && \
cp -f /usr/lib/libjansson.so* /usr/lib64/ && \
wget -O /root/cmake-3.25.1-linux-x86_64.tar.gz https://artifactory.hashdata.xyz/artifactory/utility/cmake-3.25.1-linux-x86_64.tar.gz && \
mkdir -p /root/cmake-3.25.1-linux-x86_64 && \
tar -xvf /root/cmake-3.25.1-linux-x86_64.tar.gz -C /root/cmake-3.25.1-linux-x86_64 && \
ln -s /root/cmake-3.25.1-linux-x86_64/cmake-3.25.1-linux-x86_64/bin/cmake /usr/bin/cmake && \
chmod u+s /bin/ping && \
mkdir -p /data0/data1/primary && \
mkdir -p /data0/data2/primary && \
Expand Down
1 change: 0 additions & 1 deletion gpMgmt/bin/pythonSrc/ext
Open in desktop
Submodule ext deleted from 135d64
Binary file added gpMgmt/bin/pythonSrc/ext/PyGreSQL-5.2.tar.gz
View file
Open in desktop
Binary file not shown.
Binary file added gpMgmt/bin/pythonSrc/ext/PyYAML-5.3.1.tar.gz
View file
Open in desktop
Binary file not shown.
Binary file added gpMgmt/bin/pythonSrc/ext/behave-1.2.6.tar.gz
View file
Open in desktop
Binary file not shown.
Binary file added gpMgmt/bin/pythonSrc/ext/mock-1.0.1.tar.gz
View file
Open in desktop
Binary file not shown.
Binary file added gpMgmt/bin/pythonSrc/ext/parse-1.8.2.tar.gz
View file
Open in desktop
Binary file not shown.
Binary file added gpMgmt/bin/pythonSrc/ext/psutil-5.7.0.tar.gz
View file
Open in desktop
Binary file not shown.
Binary file added gpMgmt/bin/pythonSrc/ext/pylint-0.21.0.tar.gz
View file
Open in desktop
Binary file not shown.
Binary file added gpMgmt/bin/pythonSrc/ext/setuptools-36.6.0.tar.gz
View file
Open in desktop
Binary file not shown.
10 changes: 2 additions & 8 deletions hd-ci/common.bash
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,15 +72,9 @@ setup_cbdb() {

download_cbdb_tar_package() {
fts_mode=$1
source "${SRC_PATH}"/cbdb-artifacts.txt
source /opt/cbdb-artifacts.txt
mkdir -p "${ROOT_PATH}"/bin_gpdb
local download_url=${fts_mode:+$(if [ "${fts_mode}" == "external_fts" ]; then echo "${fts_mode}_"; fi)}TARGZ_cbdb_${OS_TYPE}_${OS_ARCH}_url


curl -sSfL \
-o "${ROOT_PATH}/bin_gpdb/bin_gpdb.tar.gz" \
-z "${ROOT_PATH}/bin_gpdb/bin_gpdb.tar.gz" \
"${!download_url}"
mv /opt/${internal_tar_download_url##*/} "/opt/bin_gpdb.tar.gz"
}

setup_env() {
Expand Down
Loading