From 92abd9ef1618ed88af39c30b2b1154635d3a1d18 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Thu, 23 Nov 2023 14:24:07 -0800 Subject: [PATCH] Test upload workflow for building odbc driver --- .github/workflows/win-build.yml | 446 ++++++++++++++++++++++++++++++++ 1 file changed, 446 insertions(+) create mode 100644 .github/workflows/win-build.yml diff --git a/.github/workflows/win-build.yml b/.github/workflows/win-build.yml new file mode 100644 index 0000000000000..8be40d48753af --- /dev/null +++ b/.github/workflows/win-build.yml @@ -0,0 +1,446 @@ +name: ODBC Driver for Windows + +on: + # always trigger + push: + pull_request: + + # TODO only trigger workflow when TS driver change is relevant + # push: + # branches: + # - main + # paths: + # - '.github/workflows/win-build.yml' + # - 'cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/**' + # pull_request: + # branches: + # - main + # paths: + # - '.github/workflows/win-build.yml' + # - 'cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/**' + +#-AL- todo add amz ts odbc driver path +env: + SIGNING_ENABLED: ${{ github.event.inputs.sign }} + CI_OUTPUT_PATH: "ci-output" + ODBC_LIB_PATH: "${{github.workspace}}/cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/build/odbc/lib" + ODBC_BIN_PATH: "${{github.workspace}}/cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/build/odbc/bin/Release" + ODBC_DEBUG_PATH: "${{github.workspace}}/cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/build/odbc/bin/Debug" + ODBC_BUILD_PATH: "${{github.workspace}}/cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/build/odbc/cmake" + VCPKG_ROOT: "c:/vcpkg" + TIMESTREAM_LOG_PATH: "${{github.workspace}}/cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/build/odbc/logs" + TIMESTREAM_LOG_LEVEL: "4" + BIG_TABLE_PAGINATION_TEST_ENABLE: "TRUE" + + # # AAD Test environment variables. Uncomment out to use GitHub secrets to enable AAD integration tests + # ENABLE_AAD_TEST: "TRUE" + # AAD_APP_ID: ${{secrets.AAD_APP_ID}} + # AAD_ROLE_ARN: ${{secrets.AAD_ROLE_ARN}} + # AAD_IDP_ARN: ${{secrets.AAD_IDP_ARN}} + # AAD_TENANT: ${{secrets.AAD_TENANT}} + # AAD_USER: ${{secrets.AAD_USER}} + # AAD_USER_PWD: ${{secrets.AAD_USER_PWD}} + # AAD_CLIENT_SECRET: ${{secrets.AAD_CLIENT_SECRET}} + + # # OKTA Test environment variables. Uncomment out to use GitHub secrets to enable AAD integration tests + # ENABLE_OKTA_TEST: "TRUE" + # OKTA_HOST: ${{secrets.OKTA_HOST}} + # OKTA_USER: ${{secrets.OKTA_USER}} + # OKTA_USER_PWD: ${{secrets.OKTA_USER_PWD}} + # OKTA_APP_ID: ${{secrets.OKTA_APP_ID}} + # OKTA_ROLE_ARN: ${{secrets.OKTA_ROLE_ARN}} + # OKTA_IDP_ARN: ${{secrets.OKTA_IDP_ARN}} + +jobs: + build-windows32: + runs-on: windows-2022 + steps: + - uses: actions/checkout@v2 + + - name: Install Chocolatey + run: | + Set-ExecutionPolicy Bypass -Scope Process -Force + [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072 + iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) + + - name: "Update path for WIX Toolset" + run: | + echo "C:\Program Files (x86)\WiX Toolset v3.11\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Get latest version of CMake + uses: lukka/get-cmake@latest + + - name: add-msbuild-to-path + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Cache dependencies Windows 32 + id: cache-dependencies-windows-32 + uses: actions/cache@v3 + with: + path: | + ${{ env.VCPKG_ROOT }} + key: ${{ runner.os }}-cache-dependencies-windows-32-v1 + + - name: Install dependencies Windows + run: vcpkg integrate install; vcpkg install boost-test:x86-windows boost-asio:x86-windows boost-chrono:x86-windows boost-interprocess:x86-windows boost-regex:x86-windows boost-system:x86-windows boost-thread:x86-windows --recurse + env: + VCPKG_ROOT: ${{ env.VCPKG_ROOT }} + + - name: configure-and-build-driver + run: | + cd cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver + .\build_win_release32.ps1 + + - name: Build PowerBI connector + run: | + cd cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/src/PowerBIConnector + .\AmazonTimestreamConnector.ps1 + cd ${{github.workspace}}/cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver + + - name: Upload ODBC connector build + uses: actions/upload-artifact@v3 + with: + name: AmazonTimestreamConnector.mez + path: ./cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/src/PowerBIConnector/AmazonTimestreamConnector.mez + + - name: register-driver + run: | + .\cpp\src\arrow\flight\sql\amazon-timestream-odbc-driver\src\odbc\install\install_amd64.cmd ${{env.ODBC_BIN_PATH}}\timestream.odbc.dll ${{env.ODBC_BIN_PATH}}\timestream.odbc.dll + + # -AL- removed + # - name: configure-aws-credentials-for-tests + # if: success() + # uses: aws-actions/configure-aws-credentials@v2 + # with: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: us-west-2 + + - name: run-unit-tests + id: rununittests + run: | + mkdir -p "${{env.TIMESTREAM_LOG_PATH}}" + ${{env.ODBC_BIN_PATH}}/timestream-odbc-unit-tests.exe + + - name: upload-unit-test-report + if: always() + uses: EnricoMi/publish-unit-test-result-action/composite@v2.7.0 + with: + check_name: "Windows x32 Build Unit Test Results Check" + comment_title: "Windows 2022 (x32) Build Unit Test Results" + files: ./odbc_unit_test_result.xml + + - name: upload-unit-test-file + if: always() + uses: actions/upload-artifact@v3 + with: + name: odbc-unit-test-results + path: | + ./odbc_unit_test_result.xml + ./cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/build/odbc/logs/timestream_odbc_*.log + + - name: run-integration-tests + id: runintegrationtests + run: | + ${{env.ODBC_BIN_PATH}}/timestream-odbc-integration-tests.exe + + - name: upload-integration-test-report + if: always() + uses: EnricoMi/publish-unit-test-result-action/composite@v2.7.0 + with: + check_name: "Windows x32 Build Integration Test Results Check" + comment_title: "Windows 2022 (x32) Build Integration Test Results" + files: ./odbc_test_result.xml + + - name: upload-integration-test-file + if: always() + uses: actions/upload-artifact@v3 + with: + name: odbc-test-results + path: | + ./odbc_test_result.xml + ./cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/build/odbc/logs/timestream_odbc_*.log + + - name: Read VERSION file + if: ${{ env.SIGNING_ENABLED == 'true' }} + id: getversion + run: echo "::set-output name=version::$(cat src/ODBC_DRIVER_VERSION.txt)" + + # - name: "Configure AWS credentials" + # if: ${{ env.SIGNING_ENABLED == 'true' }} + # uses: aws-actions/configure-aws-credentials@v2 + # with: + # role-skip-session-tagging: true + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: us-west-2 + # role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + # role-external-id: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} + # role-duration-seconds: 3600 + + # # Name of signed executable is same as unsigned exeuctable + # - name: Sign executables + # if: ${{ env.SIGNING_ENABLED == 'true' }} + # id: sign-executables-32 + # shell: pwsh + # run: | + # . ./scripts/sign_installer.ps1 + # choco install jq -y + # Invoke-SignInstaller build\odbc\cmake\Release x86 ${{ steps.getversion.outputs.version }} ${{ secrets.AWS_UNSIGNED_BUCKET }} ${{ secrets.AWS_SIGNED_BUCKET }} ${{ secrets.AWS_KEY }} + + - name: Upload ODBC driver build + uses: actions/upload-artifact@v3 + with: + name: timestream-odbc-driver-32 + path: cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/build/odbc/cmake/Release + + build-windows64: + runs-on: windows-2022 + steps: + - uses: actions/checkout@v2 + + - name: Install Chocolatey + run: | + Set-ExecutionPolicy Bypass -Scope Process -Force + [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072 + iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) + + - name: "Update path for WIX Toolset" + run: | + echo "C:\Program Files (x86)\WiX Toolset v3.11\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Get latest version of CMake + uses: lukka/get-cmake@latest + + - name: add-msbuild-to-path + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Cache dependencies Windows 64 + id: cache-dependencies-windows-64 + uses: actions/cache@v3 + with: + path: | + ${{ env.VCPKG_ROOT }} + key: ${{ runner.os }}-cache-dependencies-windows-64-v1 + + - name: Install dependencies Windows + run: vcpkg integrate install; vcpkg install boost-test:x64-windows boost-asio:x64-windows boost-chrono:x64-windows boost-interprocess:x64-windows boost-regex:x64-windows boost-system:x64-windows boost-thread:x64-windows --recurse + env: + VCPKG_ROOT: ${{env.VCPKG_ROOT}} + + - name: configure-and-build-driver + run: | + cd cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver + .\build_win_release64.ps1 + + - name: Build PowerBI connector + run: | + cd cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/src/PowerBIConnector + .\AmazonTimestreamConnector.ps1 + cd ${{github.workspace}}/cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver + + - name: Upload ODBC connector build + uses: actions/upload-artifact@v3 + with: + name: AmazonTimestreamConnector.mez + path: ./cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/src/PowerBIConnector/AmazonTimestreamConnector.mez + + - name: register-driver + run: | + .\cpp\src\arrow\flight\sql\amazon-timestream-odbc-driver\src\odbc\install\install_amd64.cmd ${{env.ODBC_BIN_PATH}}\timestream.odbc.dll + + # - name: configure-aws-credentials-for-tests + # if: success() + # uses: aws-actions/configure-aws-credentials@v2 + # with: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: us-west-2 + + - name: run-unit-tests + id: rununittests + run: | + mkdir -p "${{env.TIMESTREAM_LOG_PATH}}" + ${{env.ODBC_BIN_PATH}}/timestream-odbc-unit-tests.exe + + - name: upload-unit-test-report + if: always() + uses: EnricoMi/publish-unit-test-result-action/composite@v2.7.0 + with: + check_name: "Windows 2022 (x64) Build Unit Test Results Check" + comment_title: "Windows 2022 (x64) Build Unit Test Results" + files: ./odbc_unit_test_result.xml + + - name: upload-unit-test-file + if: always() + uses: actions/upload-artifact@v3 + with: + name: odbc-unit-test-results + path: | + ./odbc_unit_test_result.xml + ./cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/build/odbc/logs/timestream_odbc_*.log + + - name: run-integration-tests + id: runintegrationtests + run: | + ${{env.ODBC_BIN_PATH}}/timestream-odbc-integration-tests.exe + + - name: upload-integration-test-report + if: always() + uses: EnricoMi/publish-unit-test-result-action/composite@v2.7.0 + with: + check_name: "Windows 2022 (x64) Build Integration Test Results Check" + comment_title: "Windows 2022 (x64) Build Integration Test Results" + files: ./odbc_test_result.xml + + - name: upload-integration-test-file + if: always() + uses: actions/upload-artifact@v3 + with: + name: odbc-test-results + path: | + ./odbc_test_result.xml + ./cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/build/odbc/logs/timestream_odbc_*.log + + - name: Read VERSION file + if: ${{ env.SIGNING_ENABLED == 'true' }} + id: getversion + run: echo "::set-output name=version::$(cat src/ODBC_DRIVER_VERSION.txt)" + + # - name: "Configure AWS credentials" + # if: ${{ env.SIGNING_ENABLED == 'true' }} + # uses: aws-actions/configure-aws-credentials@v2 + # with: + # role-skip-session-tagging: true + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: us-west-2 + # role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + # role-external-id: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} + # role-duration-seconds: 3600 + + # Name of signed executable is same as unsigned exeuctable + # - name: Sign executables + # if: ${{ env.SIGNING_ENABLED == 'true' }} + # id: sign-executables-64 + # shell: pwsh + # run: | + # . ./scripts/sign_installer.ps1 + # choco install jq -y + # Invoke-SignInstaller build\odbc\cmake\Release x64 ${{ steps.getversion.outputs.version }} ${{ secrets.AWS_UNSIGNED_BUCKET }} ${{ secrets.AWS_SIGNED_BUCKET }} ${{ secrets.AWS_KEY }} + + - name: Upload ODBC driver build + uses: actions/upload-artifact@v3 + with: + name: timestream-odbc-driver-64 + path: cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/build/odbc/cmake/Release + + build-windows64_coverage: + runs-on: windows-2022 + steps: + - uses: actions/checkout@v2 + + - name: Install Chocolatey + run: | + Set-ExecutionPolicy Bypass -Scope Process -Force + [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072 + iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) + + - name: "Update path for WIX Toolset" + run: | + echo "C:\Program Files (x86)\WiX Toolset v3.11\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Get latest version of CMake + uses: lukka/get-cmake@latest + + - name: add-msbuild-to-path + uses: microsoft/setup-msbuild@v1.0.2 + + - name: setup-opencppcoverage-and-add-to-path + id: setup_opencppcoverage + run: | + choco install OpenCppCoverage -y + echo "C:\Program Files\OpenCppCoverage" >> $env:GITHUB_PATH + + - name: Cache dependencies Windows 64 + id: cache-dependencies-windows-64 + uses: actions/cache@v3 + with: + path: | + ${{ env.VCPKG_ROOT }} + key: ${{ runner.os }}-cache-dependencies-windows-64-v1 + + - name: Install dependencies Windows + run: vcpkg integrate install; vcpkg install boost-test:x64-windows boost-asio:x64-windows boost-chrono:x64-windows boost-interprocess:x64-windows boost-regex:x64-windows boost-system:x64-windows boost-thread:x64-windows --recurse + env: + VCPKG_ROOT: ${{env.VCPKG_ROOT}} + + - name: configure-and-build-driver + run: | + cd cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver + .\build_win_debug64.ps1 + + - name: Build PowerBI connector + run: | + cd cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/src/PowerBIConnector + .\AmazonTimestreamConnector.ps1 + cd ${{github.workspace}}/cpp/src/arrow/flight/sql/amazon-timestream-odbc-driver/ + + - name: register-driver + run: | + .\cpp\src\arrow\flight\sql\amazon-timestream-odbc-driver\src\odbc\install\install_amd64.cmd ${{env.ODBC_DEBUG_PATH}}\timestream.odbc.dll + + # - name: configure-aws-credentials-for-tests + # if: success() + # uses: aws-actions/configure-aws-credentials@v2 + # with: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: us-west-2 + + - name: run-tests + run: | + # run code coverage against unit test first + opencppcoverage --sources \cpp\src\arrow\flight\sql\amazon-timestream-odbc-driver\src\odbc\src --sources \cpp\src\arrow\flight\sql\amazon-timestream-odbc-driver\src\odbc\include --export_type binary:unit.test.cov --excluded_modules boost_unit_test_framework --excluded_modules timestream-odbc-integration-tests --excluded_modules zlib --excluded_modules aws --excluded_modules boost --excluded_sources \cpp\src\arrow\flight\sql\amazon-timestream-odbc-driver\src\odbc\src\ignite --excluded_sources \cpp\src\arrow\flight\sql\amazon-timestream-odbc-driver\src\odbc\include\ignite -- ${{env.ODBC_DEBUG_PATH}}/timestream-odbc-unit-tests.exe + # run code coverage against integration test and merge the result with the unit test one + opencppcoverage --sources \cpp\src\arrow\flight\sql\amazon-timestream-odbc-driver\src\odbc\src --sources \cpp\src\arrow\flight\sql\amazon-timestream-odbc-driver\src\odbc\include --input_coverage unit.test.cov --export_type cobertura:coverage.cobertura.xml --excluded_modules boost_unit_test_framework --excluded_modules timestream-odbc-integration-tests --excluded_modules zlib --excluded_modules aws --excluded_modules boost --excluded_sources \cpp\src\arrow\flight\sql\amazon-timestream-odbc-driver\src\odbc\src\ignite --excluded_sources \cpp\src\arrow\flight\sql\amazon-timestream-odbc-driver\src\odbc\include\ignite -- ${{env.ODBC_DEBUG_PATH}}/timestream-odbc-integration-tests.exe + + - name: upload-coverage + if: always() + uses: actions/upload-artifact@v3 + with: + name: code-coverage + path: coverage.cobertura.xml + + comment_PR_coverage: + runs-on: ubuntu-latest + needs: build-windows64_coverage + steps: + - uses: actions/checkout@v2 + + - name: Retrieve coverage + uses: actions/download-artifact@v2 + with: + name: code-coverage + + - name: Code Coverage Summary Report + uses: irongut/CodeCoverageSummary@v1.2.0 + with: + filename: coverage.cobertura.xml + badge: true + format: markdown + indicators: true + output: both + + - name: Add Header for Code Coverage Summary Report + run: | + echo "## Windows 2022 (x64) Debug Build Code Coverage Test Results Check" > coverage-cobertura.md + cat code-coverage-results.md >> coverage-cobertura.md + + - name: Add Coverage PR Comment + uses: marocchino/sticky-pull-request-comment@v2 + if: github.event_name == 'pull_request' + with: + header: windows + recreate: true + path: coverage-cobertura.md