Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dev: reduce log in gradle commands #12677

Merged
merged 1 commit into from
Feb 20, 2025
Merged

dev: reduce log in gradle commands #12677

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions metadata-ingestion-modules/airflow-plugin/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ ext {
venv_name = 'venv'
}

ext.venv_activate_command = "set +x && source ${venv_name}/bin/activate && set -x && "
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might be nicer to leave off the final && and put that in the usage, so that it reads like a normal shell command


if (!project.hasProperty("extra_pip_requirements")) {
ext.extra_pip_requirements = ""
}
Expand Down Expand Up @@ -37,7 +39,7 @@ task installPackage(type: Exec, dependsOn: [environmentSetup, ':metadata-ingesti
inputs.file file('setup.py')
outputs.file(sentinel_file)
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"${pip_install_command} -e .[ignore${extra_pip_extras}] ${extra_pip_requirements} &&" +
"touch ${sentinel_file}"
}
Expand All @@ -51,7 +53,7 @@ task installDev(type: Exec, dependsOn: [install]) {
// For some reason, the Airflow constraints files pin things like black and pytest.
// This allows us to not respect those constraints ourselves.
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"${pip_install_command} -e .[dev${extra_pip_extras}] && " +
"touch ${sentinel_file}"
}
Expand All @@ -60,7 +62,7 @@ task installTest(type: Exec, dependsOn: [installDev]) {
inputs.file file('setup.py')
outputs.file(sentinel_file)
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"${pip_install_command} -e .[integration-tests${extra_pip_extras}] ${extra_pip_requirements} && " +
"touch ${sentinel_file}"
}
Expand All @@ -72,14 +74,14 @@ task lint(type: Exec, dependsOn: installDev) {
*/
commandLine 'bash', '-c',
"find ${venv_name}/lib -path *airflow/_vendor/connexion/spec.py -exec sed -i.bak -e '169,169s/ # type: List\\[str\\]//g' {} \\; && " +
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"ruff check src/ tests/ && " +
"ruff format --check src/ tests/ && " +
"mypy --show-traceback --show-error-codes src/ tests/"
}
task lintFix(type: Exec, dependsOn: installDev) {
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"ruff check --fix src/ tests/ && " +
"ruff format src/ tests/ "
}
Expand All @@ -95,7 +97,7 @@ task testQuick(type: Exec, dependsOn: installTest) {
inputs.files(project.fileTree(dir: "src/", include: "**/*.py"))
inputs.files(project.fileTree(dir: "tests/"))
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"pytest --cov-config=setup.cfg ${get_coverage_args('quick')} -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
}

Expand All @@ -106,7 +108,7 @@ task cleanPythonCache(type: Exec) {
}
task buildWheel(type: Exec, dependsOn: [environmentSetup]) {
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
'uv pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_INSTALL=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
}

Expand Down
42 changes: 22 additions & 20 deletions metadata-ingestion/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ ext {
venv_name = 'venv'
}

ext.venv_activate_command = "set +x && source ${venv_name}/bin/activate && set -x && "

if (!project.hasProperty("extra_pip_requirements")) {
ext.extra_pip_requirements = ""
}
Expand All @@ -35,7 +37,7 @@ task installPackageOnly(type: Exec, dependsOn: environmentSetup) {
inputs.file file('setup.py')
outputs.file(sentinel_file)
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"uv pip install -e . &&" +
"touch ${sentinel_file}"
}
Expand All @@ -45,7 +47,7 @@ task installPackage(type: Exec, dependsOn: installPackageOnly) {
inputs.file file('setup.py')
outputs.file(sentinel_file)
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"uv pip install -e . ${extra_pip_requirements} && " +
"touch ${sentinel_file}"
}
Expand All @@ -56,14 +58,14 @@ task codegen(type: Exec, dependsOn: [environmentSetup, installPackage, ':metadat
project.fileTree(dir: "scripts"),
)
outputs.dir('src/datahub/metadata')
commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/codegen.sh"
commandLine 'bash', '-c', "${venv_activate_command} ./scripts/codegen.sh"
}

task customPackageGenerate(type: Exec, dependsOn: [environmentSetup, installPackage, ':metadata-events:mxe-schemas:build']) {
def package_name = project.findProperty('package_name')
def package_version = project.findProperty('package_version')
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"uv pip install build && " +
"./scripts/custom_package_codegen.sh '${package_name}' '${package_version}'"
}
Expand All @@ -75,7 +77,7 @@ task installDev(type: Exec, dependsOn: [install]) {
inputs.file file('setup.py')
outputs.file(sentinel_file)
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"uv pip install -e .[dev] ${extra_pip_requirements} && " +
"touch ${sentinel_file}"
}
Expand All @@ -85,7 +87,7 @@ task installAll(type: Exec, dependsOn: [install]) {
inputs.file file('setup.py')
outputs.file(sentinel_file)
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"uv pip install -e .[all] ${extra_pip_requirements} && " +
"touch ${sentinel_file}"
}
Expand All @@ -99,25 +101,25 @@ task modelDocGen(type: Exec, dependsOn: [codegen]) {
project.fileTree(dir: "../metadata-events/mxe-schemas/src/", include: "**/*.avsc")
)
outputs.dir('../docs/generated/metamodel')
commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/modeldocgen.sh"
commandLine 'bash', '-c', "${venv_activate_command} ./scripts/modeldocgen.sh"
}

task modelDocUpload(type: Exec, dependsOn: [modelDocGen]) {
commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/modeldocupload.sh"
commandLine 'bash', '-c', "${venv_activate_command} ./scripts/modeldocupload.sh"
}


task lint(type: Exec, dependsOn: installDev) {
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"ruff check src/ tests/ examples/ && " +
"ruff format --check src/ tests/ examples/ && " +
"mypy --show-traceback --show-error-codes src/ tests/ examples/"
}

task lintFix(type: Exec, dependsOn: installDev) {
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"ruff check --fix src/ tests/ examples/ && " +
"ruff format src/ tests/ examples/ "
}
Expand All @@ -132,7 +134,7 @@ task testQuick(type: Exec, dependsOn: [installDev, ':metadata-models:generateJso
outputs.dir("${venv_name}")
def cvg_arg = get_coverage_args("quick")
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"${pytest_default_env} pytest ${cvg_arg} tests/unit ${pytest_default_args} --random-order -m 'not integration' --junit-xml=junit.quick.xml"
}

Expand All @@ -142,7 +144,7 @@ task installDevTest(type: Exec, dependsOn: [install]) {
outputs.dir("${venv_name}")
outputs.file(sentinel_file)
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"uv pip install -e .[dev,integration-tests] ${extra_pip_requirements} && " +
"touch ${sentinel_file}"
}
Expand All @@ -153,7 +155,7 @@ task testSingle(dependsOn: [installDevTest]) {
if (testFile != 'unknown') {
exec {
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && ${pytest_default_env} pytest ${testFile} ${pytest_default_args}"
"${venv_activate_command} ${pytest_default_env} pytest ${testFile} ${pytest_default_args}"
}
} else {
throw new GradleException("No file provided. Use -PtestFile=<test_file>")
Expand All @@ -164,33 +166,33 @@ task testSingle(dependsOn: [installDevTest]) {
task testIntegrationBatch0(type: Exec, dependsOn: [installDevTest]) {
def cvg_arg = get_coverage_args("intBatch0")
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"${pytest_default_env} pytest ${cvg_arg} ${pytest_default_args} -m 'integration_batch_0' --junit-xml=junit.integrationbatch0.xml"
}
task testIntegrationBatch1(type: Exec, dependsOn: [installDevTest]) {
def cvg_arg = get_coverage_args("intBatch1")
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"${pytest_default_env} pytest ${cvg_arg} ${pytest_default_args} -m 'integration_batch_1' --junit-xml=junit.integrationbatch1.xml"
}
task testIntegrationBatch2(type: Exec, dependsOn: [installDevTest]) {
def cvg_arg = get_coverage_args("intBatch2")
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"${pytest_default_env} pytest ${cvg_arg} ${pytest_default_args} -m 'integration_batch_2' --junit-xml=junit.integrationbatch2.xml"
}

task testFull(type: Exec, dependsOn: [installDevTest]) {
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " +
venv_activate_command +
"${pytest_default_env} pytest ${pytest_default_args} --junit-xml=junit.full.xml"
}

task specGen(type: Exec, dependsOn: [codegen, installDevTest]) {
commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/specgen.sh"
commandLine 'bash', '-c', "${venv_activate_command} ./scripts/specgen.sh"
}
task docGen(type: Exec, dependsOn: [codegen, installDevTest, specGen]) {
commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/docgen.sh"
commandLine 'bash', '-c', "${venv_activate_command} ./scripts/docgen.sh"
}


Expand All @@ -201,7 +203,7 @@ task cleanPythonCache(type: Exec) {
}
task buildWheel(type: Exec, dependsOn: [install, codegen, cleanPythonCache]) {
commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && " +
venv_activate_command +
'uv pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_INSTALL=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
}

Expand Down
Loading