clustering: User defined distances and keyers (#6612) #25
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Snapshot release | |
| on: | |
| push: | |
| branches: | |
| - 'master' | |
| - '4.0' | |
| paths-ignore: | |
| - 'docs/**' | |
| release: | |
| types: [created] | |
| jobs: | |
| prepare_e2e_test_matrix: | |
| if: github.repository == 'OpenRefine/OpenRefine' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| - id: set-matrix | |
| run: npm install --save glob@8.1.0 && node main/tests/cypress/build-test-matrix.js >> $GITHUB_OUTPUT | |
| env: | |
| browsers: chrome | |
| e2e_test: | |
| if: github.repository == 'OpenRefine/OpenRefine' | |
| name: e2e_test ${{ matrix.specs.group }} | |
| needs: prepare_e2e_test_matrix | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: ${{fromJSON(needs.prepare_e2e_test_matrix.outputs.matrix)}} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - name: Set up secrets | |
| run: | | |
| echo "CYPRESS_RECORD_KEY=$(echo YzE3ZDU4OGItZTBkOC00ZjJmLTg4NjYtNzJmNmFmYmRhNGQxCg== | base64 -d)" >> $GITHUB_ENV | |
| echo "CYPRESS_PROJECT_ID=s5du3k" >> $GITHUB_ENV | |
| - name: Set up Java 11 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 11 | |
| cache: 'maven' | |
| - name: Build OpenRefine | |
| run: ./refine build | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| - name: Restore Tests dependency cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/cache | |
| ~/.cache | |
| **/node_modules | |
| !~/cache/exclude | |
| key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn | |
| - name: Install test dependencies | |
| run: | | |
| cd ./main/tests/cypress | |
| npm i -g yarn | |
| yarn install | |
| - name: Test with Cypress on ${{ matrix.browser }} | |
| run: | | |
| echo REFINE_MIN_MEMORY=1400M >> ./refine.ini | |
| echo REFINE_MEMORY=4096M >> ./refine.ini | |
| ./refine e2e_tests | |
| env: | |
| CYPRESS_BROWSER: ${{ matrix.browser }} | |
| CYPRESS_RECORD_KEY: ${{ env.CYPRESS_RECORD_KEY }} | |
| CYPRESS_PROJECT_ID: ${{ env.CYPRESS_PROJECT_ID }} | |
| CYPRESS_CI_BUILD_ID: '${{ github.run_id }}' | |
| CYPRESS_SPECS: ${{ matrix.specs.paths }} | |
| CYPRESS_GROUP: ${{ matrix.specs.group }} | |
| build: | |
| if: github.repository == 'OpenRefine/OpenRefine' | |
| services: | |
| postgres: | |
| image: postgres | |
| ports: | |
| - 5432 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: 'postgres' | |
| POSTGRES_DB: test_db | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| mysql: | |
| image: mysql:8 | |
| ports: | |
| - 3306 | |
| env: | |
| MYSQL_ROOT_PASSWORD: root | |
| options: >- | |
| --health-cmd "mysqladmin ping" | |
| --health-interval 5s | |
| --health-timeout 2s | |
| --health-retries 3 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # This is wasteful, but needed for git describe | |
| - name: Set up secrets | |
| run: | | |
| echo "COVERALLS_TOKEN=$(echo eUVUVGRHOFJhQm9GMFJBYTNibjVhcWFEblpac1lmMlE3Cg== | base64 -d)" >> $GITHUB_ENV | |
| - name: Set up Java 11 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 11 | |
| cache: 'maven' | |
| server-id: ossrh | |
| server-username: OSSRH_USER | |
| server-password: OSSRH_PASS | |
| - name: Install genisoimage and jq | |
| run: sudo apt-get install genisoimage jq | |
| - name: Configure connections to databases | |
| id: configure_db_connections | |
| run: cat extensions/database/tests/conf/github_actions_tests.xml | sed -e "s/MYSQL_PORT/${{ job.services.mysql.ports[3306] }}/g" | sed -e "s/POSTGRES_PORT/${{ job.services.postgres.ports[5432] }}/g" > extensions/database/tests/conf/tests.xml | |
| - name: Populate databases with test data | |
| id: populate_databases_with_test_data | |
| run: | | |
| mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot -e 'CREATE DATABASE test_db;' | |
| mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot < extensions/database/tests/conf/test-mysql.sql | |
| psql -U postgres test_db -h 127.0.0.1 -p ${{ job.services.postgres.ports[5432] }} < extensions/database/tests/conf/test-pgsql.sql | |
| env: | |
| PGPASSWORD: postgres | |
| - name: Install webapp dependencies | |
| run: cd main/webapp && npm install | |
| - name: Build and test with Maven | |
| run: mvn -B jacoco:prepare-agent test jacoco:report | |
| - name: Coveralls | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| github-token: ${{ env.COVERALLS_TOKEN }} | |
| format: jacoco | |
| flag-name: Java ${{ matrix.java }} | |
| pull-request: ${{ github.event.number }} | |
| fail-on-error: false | |
| - name: Install webapp dependencies | |
| run: cd main/webapp && npm install | |
| - name: Generate dist files | |
| run: mvn -B package -DskipTests=true | |
| mac_test_and_deploy: | |
| if: github.repository == 'OpenRefine/OpenRefine' | |
| runs-on: macos-12 | |
| outputs: | |
| version_number_string: ${{ steps.version_string_variable.outputs.output_value }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Java 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 17 | |
| cache: 'maven' | |
| server-id: ossrh | |
| server-username: OSSRH_USER | |
| server-password: OSSRH_PASS | |
| gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} | |
| gpg-passphrase: GPG_PASSPHRASE | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3' | |
| - name: Install dmgbuild | |
| run: pip install dmgbuild | |
| - name: Decrypt Apple certificates | |
| run: | | |
| gpg --pinentry-mode loopback --passphrase "${{ secrets.GPG_PASSPHRASE }}" --output .github/workflows/release/apple_cert.cer --decrypt packaging/apple_certs/apple_cert.cer.enc | |
| gpg --pinentry-mode loopback --passphrase "${{ secrets.GPG_PASSPHRASE }}" --output .github/workflows/release/apple_cert.p12 --decrypt packaging/apple_certs/apple_cert.p12.enc | |
| .github/workflows/release/add_apple_keys.sh | |
| env: | |
| P12_PASSPHRASE: ${{ secrets.APPLE_P12_PASSPHRASE }} | |
| - name: Apply patch to insert Google API credentials | |
| run: | | |
| gpg --pinentry-mode loopback --passphrase "${{ secrets.GPG_PASSPHRASE }}" --output .github/workflows/release/gdata.patch --decrypt .github/workflows/release/gdata.patch.asc | |
| git am .github/workflows/release/gdata.patch | |
| git reset HEAD^ | |
| - name: Install webapp dependencies | |
| run: cd main/webapp && npm install | |
| - name: Get release upload URL | |
| id: get_release_upload_url | |
| if: github.event_name == 'release' | |
| uses: bruceadams/get-release@v1.3.2 | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| - name: Set version | |
| if: github.event_name == 'release' | |
| run: mvn versions:set -DnewVersion=$(echo '${{ steps.get_release_upload_url.outputs.tag_name }}' | sed 's/[^a-zA-Z0-9_\.\-]/_/g') | |
| - name: Build and test | |
| run: mvn -B test | |
| env: | |
| GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} | |
| - name: Set version string output variable | |
| id: version_string_variable | |
| run: echo "output_value=$(mvn org.apache.maven.plugins:maven-help-plugin:3.1.0:evaluate -Dexpression=project.version -q -DforceStdout)" >> "$GITHUB_OUTPUT" | |
| - name: Package and upload to OSSRH | |
| run: | | |
| mvn -B deploy -DskipTests=true -DrepositoryId=ossrh -Dapple.notarization.username=$APPLE_USERNAME -Dapple.notarization.password=$APPLE_PASSWORD -Dapple.notarization.team.id=$APPLE_TEAM_ID -Dgpg.signer=bc | |
| env: | |
| OSSRH_USER: ${{ secrets.OSSRH_USER }} | |
| OSSRH_PASS: ${{ secrets.OSSRH_PASS }} | |
| MAVEN_GPG_KEY: ${{ secrets.GPG_PRIVATE_KEY }} | |
| MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} | |
| APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} | |
| APPLE_USERNAME: ${{ secrets.APPLE_USERNAME }} | |
| APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }} | |
| - name: Upload release asset for Windows (with Java) | |
| id: upload-release-asset-win-with-java | |
| if: github.event_name == 'release' | |
| uses: shogo82148/actions-upload-release-asset@v1.7.5 | |
| with: | |
| upload_url: ${{ steps.get_release_upload_url.outputs.upload_url }} | |
| asset_path: ./packaging/target/openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}.zip | |
| asset_name: openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}.zip | |
| asset_content_type: application/zip | |
| - name: Upload build artifact for Windows (with Java) | |
| id: upload-build-artifact-win-with-java | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: "./packaging/target/openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}.zip" | |
| name: openrefine-win-with-java-archive-${{ steps.version_string_variable.outputs.output_value }} | |
| - name: Unzip build artifact for Windows (with Java) for Installer | |
| id: unzip-build-artifact-win-with-java | |
| run: unzip ./packaging/target/openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}.zip -d ./packaging/target/openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }} | |
| - name: Upload build artifact for Windows (with Java) for Installer | |
| id: upload-build-artifact-win-with-java-for-installer | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: installertemp-win-with-java-${{ steps.version_string_variable.outputs.output_value }} | |
| path: ./packaging/target/openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}/openrefine-${{ steps.version_string_variable.outputs.output_value }} | |
| - name: Upload Release Asset Linux | |
| id: upload-release-asset-linux | |
| if: github.event_name == 'release' | |
| uses: shogo82148/actions-upload-release-asset@v1.7.5 | |
| with: | |
| upload_url: ${{ steps.get_release_upload_url.outputs.upload_url }} | |
| asset_path: ./packaging/target/openrefine-linux-${{ steps.version_string_variable.outputs.output_value }}.tar.gz | |
| asset_name: openrefine-linux-${{ steps.version_string_variable.outputs.output_value }}.tar.gz | |
| asset_content_type: application/tar+gzip | |
| - name: Upload build artifact for Linux | |
| id: upload-build-artifact-linux | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: "./packaging/target/openrefine-linux-${{ steps.version_string_variable.outputs.output_value }}.tar.gz" | |
| name: openrefine-linux-${{ steps.version_string_variable.outputs.output_value }} | |
| - name: Upload Release Asset Mac | |
| id: upload-release-asset-mac | |
| if: github.event_name == 'release' | |
| uses: shogo82148/actions-upload-release-asset@v1.7.5 | |
| with: | |
| upload_url: ${{ steps.get_release_upload_url.outputs.upload_url }} | |
| asset_path: ./packaging/target/openrefine-mac-${{ steps.version_string_variable.outputs.output_value }}.dmg | |
| asset_name: openrefine-mac-${{ steps.version_string_variable.outputs.output_value }}.dmg | |
| asset_content_type: application/x-apple-diskimage | |
| - name: Upload build artifact for Mac | |
| id: upload-build-artifact-mac | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: "./packaging/target/openrefine-mac-${{ steps.version_string_variable.outputs.output_value }}.dmg" | |
| name: openrefine-mac-${{ steps.version_string_variable.outputs.output_value }} | |
| windows_installer: | |
| if: github.repository == 'OpenRefine/OpenRefine' | |
| runs-on: windows-2022 | |
| needs: mac_test_and_deploy | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get release upload URL | |
| id: get_release_upload_url | |
| if: github.event_name == 'release' | |
| uses: bruceadams/get-release@v1.3.2 | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| - name: Download build artifact for Windows Installer (with Java) | |
| id: download-build-artifact-win-with-java | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: installertemp-win-with-java-${{ needs.mac_test_and_deploy.outputs.version_number_string }} | |
| path: ./packaging/target/openrefine-win-with-java-${{ needs.mac_test_and_deploy.outputs.version_number_string }} | |
| # - name: Sign the executables | |
| # uses: nextgens/authenticode-sign-action@v1.0.1 | |
| # with: | |
| # certificate: '${{ secrets.CERTIFICATES }}' | |
| # credentials: '${{ secrets.CERTIFICATE_CREDENTIALS }}' | |
| # key-uri: "projects/myProject/locations/europe-west2/keyRings/code-signing/cryptoKeys/ev/cryptoKeyVersions/1" | |
| # timestamp-url: 'http://timestamp.digicert.com' | |
| # description: 'OpenRefine' | |
| # description-url: 'https://openrefine.org' | |
| # folder: "./packaging/target/openrefine-win-with-java-${{ needs.mac_test_and_deploy.outputs.version_number_string }}" | |
| # recursive: true | |
| - name: Build self-containing installer with InnoSetup 6 for Windows | |
| id: build-installer-innosetup-win-with-java | |
| run: | | |
| "%ProgramFiles(x86)%\Inno Setup 6\iscc.exe" -DMyAppVersion="${{ needs.mac_test_and_deploy.outputs.version_number_string }}" -DMyProgramFiles="target\openrefine-win-with-java-${{ needs.mac_test_and_deploy.outputs.version_number_string }}" "packaging\openrefine.iss" | |
| shell: cmd | |
| - name: Cleanup unused installer build artifact (delete) | |
| id: delete-build-artifact-win-with-java | |
| uses: geekyeggo/delete-artifact@v5 | |
| with: | |
| name: installertemp-win-with-java-${{ needs.mac_test_and_deploy.outputs.version_number_string }} | |
| useGlob: false | |
| failOnError: false | |
| # - name: Sign the installer | |
| # uses: nextgens/authenticode-sign-action@v1.0.1 | |
| # with: | |
| # certificate: '${{ secrets.CERTIFICATES }}' | |
| # credentials: '${{ secrets.CERTIFICATE_CREDENTIALS }}' | |
| # key-uri: "projects/myProject/locations/europe-west2/keyRings/code-signing/cryptoKeys/ev/cryptoKeyVersions/1" | |
| # timestamp-url: 'http://timestamp.digicert.com' | |
| # description: 'OpenRefine Installer' | |
| # description-url: 'https://openrefine.org' | |
| # folder: "./packaging/target" | |
| # recursive: false | |
| # - name: Verify the signature | |
| # run: call "%ProgramFiles(x86)%\Windows Kits\10\bin\10.0.17763.0\x86\signtool.exe" verify /v /pa "OpenRefine-Installer-${{ needs.mac_test_and_deploy.outputs.version_number_string }}.exe" | |
| # shell: cmd | |
| - name: Upload the installer as an artifact | |
| id: upload-build-artifact-win-with-java-installer | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: "./packaging/target/openrefine-win-with-java-installer-${{ needs.mac_test_and_deploy.outputs.version_number_string }}.exe" | |
| name: openrefine-win-with-java-installer-${{ needs.mac_test_and_deploy.outputs.version_number_string }} | |
| - name: Upload release asset for Windows (with Java) | |
| id: upload-release-asset-win-with-java-installer | |
| if: github.event_name == 'release' | |
| uses: shogo82148/actions-upload-release-asset@v1.7.5 | |
| with: | |
| upload_url: ${{ steps.get_release_upload_url.outputs.upload_url }} | |
| asset_path: ./packaging/target/openrefine-win-with-java-installer-${{ needs.mac_test_and_deploy.outputs.version_number_string }}.exe | |
| asset_name: openrefine-win-with-java-installer-${{ needs.mac_test_and_deploy.outputs.version_number_string }}.exe | |
| asset_content_type: application/vnd.microsoft.portable-executable |