From 3e6c78f08d39a143593bff27466906b76b6f1a6b Mon Sep 17 00:00:00 2001 From: Eric Wulff Date: Thu, 9 Jun 2022 10:36:38 +0200 Subject: [PATCH 1/2] feat: comet-ml logging without internet access Use the flag --comet-offline when starting a training to save comet-ml logs locally instead of streaming them to www.comet.ml. Logs can be uploaded at a later point. --- mlpf/pipeline.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mlpf/pipeline.py b/mlpf/pipeline.py index 79ce30e91..72addc922 100644 --- a/mlpf/pipeline.py +++ b/mlpf/pipeline.py @@ -103,7 +103,8 @@ def main(): @click.option("-p", "--prefix", default="", help="prefix to put at beginning of training dir name", type=str) @click.option("--plot-freq", default=None, help="plot detailed validation every N epochs", type=int) @click.option("--customize", help="customization function", type=str, default=None) -def train(config, weights, ntrain, ntest, nepochs, recreate, prefix, plot_freq, customize): +@click.option("--comet-offline", help="log comet-ml experiment locally", is_flag=True) +def train(config, weights, ntrain, ntest, nepochs, recreate, prefix, plot_freq, customize, comet_offline): #tf.debugging.enable_check_numerics() @@ -122,7 +123,14 @@ def train(config, weights, ntrain, ntest, nepochs, recreate, prefix, plot_freq, outdir = create_experiment_dir(prefix=prefix + config_file_stem + "_", suffix=platform.node()) try: - from comet_ml import Experiment + if comet_offline: + print("Using comet-ml OfflineExperiment, saving logs locally.") + from comet_ml import OfflineExperiment as Experiment + offline_dir = outdir + "/cometml" + else: + print("Using comet-ml Experiment, streaming logs to www.comet.ml.") + from comet_ml import Experiment + offline_dir = None experiment = Experiment( project_name="particleflow-tf", auto_metric_logging=True, @@ -130,6 +138,7 @@ def train(config, weights, ntrain, ntest, nepochs, recreate, prefix, plot_freq, auto_histogram_weight_logging=True, auto_histogram_gradient_logging=False, auto_histogram_activation_logging=False, + offline_directory=offline_dir, #offline_directory=outdir, #disabled=True ) From efc1e1638e53a79ca4c4da4c4fd0bdc7ae95f55a Mon Sep 17 00:00:00 2001 From: Eric Wulff Date: Thu, 9 Jun 2022 10:49:15 +0200 Subject: [PATCH 2/2] fix: comet-ml Experiment bug --- mlpf/pipeline.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/mlpf/pipeline.py b/mlpf/pipeline.py index 72addc922..6ac53f3a9 100644 --- a/mlpf/pipeline.py +++ b/mlpf/pipeline.py @@ -125,23 +125,28 @@ def train(config, weights, ntrain, ntest, nepochs, recreate, prefix, plot_freq, try: if comet_offline: print("Using comet-ml OfflineExperiment, saving logs locally.") - from comet_ml import OfflineExperiment as Experiment - offline_dir = outdir + "/cometml" + from comet_ml import OfflineExperiment + experiment = OfflineExperiment( + project_name="particleflow-tf", + auto_metric_logging=True, + auto_param_logging=True, + auto_histogram_weight_logging=True, + auto_histogram_gradient_logging=False, + auto_histogram_activation_logging=False, + offline_directory=outdir + "/cometml", + ) else: print("Using comet-ml Experiment, streaming logs to www.comet.ml.") from comet_ml import Experiment offline_dir = None - experiment = Experiment( - project_name="particleflow-tf", - auto_metric_logging=True, - auto_param_logging=True, - auto_histogram_weight_logging=True, - auto_histogram_gradient_logging=False, - auto_histogram_activation_logging=False, - offline_directory=offline_dir, - #offline_directory=outdir, - #disabled=True - ) + experiment = Experiment( + project_name="particleflow-tf", + auto_metric_logging=True, + auto_param_logging=True, + auto_histogram_weight_logging=True, + auto_histogram_gradient_logging=False, + auto_histogram_activation_logging=False, + ) except Exception as e: print("Failed to initialize comet-ml dashboard") experiment = None