From 5dfa676adb5e0d9d62fe3590e90b803492cd7be0 Mon Sep 17 00:00:00 2001 From: dberenbaum Date: Tue, 5 Mar 2024 09:20:20 -0500 Subject: [PATCH] handle auto push errors --- dvc/repo/experiments/executor/base.py | 23 ++++++++++++++++------- tests/func/experiments/test_remote.py | 13 +++++++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 86a7f048ad..933353b57d 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -677,7 +677,13 @@ def auto_push(cls, dvc: "Repo") -> Iterator[None]: git_remote = os.getenv( DVC_EXP_GIT_REMOTE, exp_config.get("git_remote", "origin") ) - cls._validate_remotes(dvc, git_remote) + try: + cls._validate_remotes(dvc, git_remote) + except DvcException as exc: + logger.warning("Failed to validate remotes. Disabling auto push: %s", exc) + + yield + return yield cls._auto_push(dvc, git_remote) @@ -689,21 +695,24 @@ def _auto_push( run_cache=True, ): from dvc.ui import ui + from dvc.utils import format_link branch = dvc.scm.get_ref(EXEC_BRANCH, follow=False) + link = format_link( + "https://dvc.org/doc/user-guide/experiment-management/sharing-experiments" + ) + ui.write( + f"Pushing experiment to '{git_remote}'. Cancel with CTRL+C. " + f"See {link} for more info." + ) try: - ui.write( - f"Auto pushing experiment to '{git_remote}'. You can cancel the push " - "with CTRL+C. If you need to push your experiment again, you can " - f"retry later using `dvc exp push`", - ) dvc.experiments.push( git_remote, branch, push_cache=push_cache, run_cache=run_cache, ) - except BaseException as exc: # noqa: BLE001 + except DvcException as exc: logger.warning( ( "Something went wrong while auto pushing experiment " diff --git a/tests/func/experiments/test_remote.py b/tests/func/experiments/test_remote.py index 62622c411e..2f064e4f86 100644 --- a/tests/func/experiments/test_remote.py +++ b/tests/func/experiments/test_remote.py @@ -409,3 +409,16 @@ def test_auto_push_on_save( dvc.experiments.save(name=exp_name, force=True) assert first(dvc.experiments.push(name=exp_name, git_remote=remote)) == expected_key + + +def test_auto_push_misconfigured( + tmp_dir, scm, dvc, git_upstream, local_remote, exp_stage, caplog +): + with dvc.config.edit() as conf: + conf["exp"]["auto_push"] = True + conf["exp"]["git_remote"] = "notfound" + + exp_name = "foo" + with caplog.at_level(logging.WARNING, logger="dvc"): + dvc.experiments.run(exp_stage.addressing, params=["foo=2"], name=exp_name) + assert "Failed to validate remotes" in caplog.text