Skip to content

feat(cli): Add run-id option to put sub-command #15873

feat(cli): Add run-id option to put sub-command

feat(cli): Add run-id option to put sub-command #15873

name: metadata ingestion
on:
push:
branches:
- master
paths:
- ".github/workflows/metadata-ingestion.yml"
- "metadata-ingestion/**"
- "metadata-ingestion-modules/**"
- "metadata-models/**"
pull_request:
branches:
- "**"
paths:
- ".github/workflows/metadata-ingestion.yml"
- "metadata-ingestion/**"
- "metadata-ingestion-modules/**"
- "metadata-models/**"
release:
types: [published]
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
metadata-ingestion:
runs-on: ubuntu-latest
env:
SPARK_VERSION: 3.3.2
DATAHUB_TELEMETRY_ENABLED: false
# TODO: Enable this once the test is fixed.
# DATAHUB_LOOKML_GIT_TEST_SSH_KEY: ${{ secrets.DATAHUB_LOOKML_GIT_TEST_SSH_KEY }}
strategy:
matrix:
python-version: ["3.8", "3.10"]
command:
[
"testQuick",
"testIntegrationBatch0",
"testIntegrationBatch1",
"testIntegrationBatch2",
]
include:
- python-version: "3.8"
- python-version: "3.10"
fail-fast: false
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 17
- uses: gradle/actions/setup-gradle@v3
- uses: acryldata/sane-checkout-action@v3
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- uses: actions/cache@v4
with:
path: |
~/.cache/uv
key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Install package
run: ./gradlew :metadata-ingestion:installPackageOnly
- name: Run lint alongwith testQuick
if: ${{ matrix.command == 'testQuick' }}
run: ./gradlew :metadata-ingestion:lint
- name: Run metadata-ingestion tests
run: ./gradlew :metadata-ingestion:${{ matrix.command }}
- name: Debug info
if: always()
run: |
source metadata-ingestion/venv/bin/activate && pip freeze
set -x
df -hl
docker image ls
docker system df
- uses: actions/upload-artifact@v3
with:
name: Test Results (metadata ingestion ${{ matrix.python-version }})
path: |
**/build/reports/tests/test/**
**/build/test-results/test/**
**/junit.*.xml
!**/binary/**
- name: Upload coverage to Codecov
if: ${{ always() && matrix.python-version == '3.10' }}
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
directory: .
fail_ci_if_error: false
flags: pytest-${{ matrix.command }}
name: pytest-${{ matrix.command }}
verbose: true
event-file:
runs-on: ubuntu-latest
steps:
- name: Upload
uses: actions/upload-artifact@v3
with:
name: Event File
path: ${{ github.event_path }}