From 6795dc829f80c0da0be13d01ce8ed2cea41fedb9 Mon Sep 17 00:00:00 2001 From: sarthakpati Date: Tue, 2 Mar 2021 08:37:47 -0500 Subject: [PATCH 01/11] added instructions to copy certificates --- docs/runningApplication.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/runningApplication.md b/docs/runningApplication.md index 1baa2736..961dde0b 100644 --- a/docs/runningApplication.md +++ b/docs/runningApplication.md @@ -143,6 +143,16 @@ wget https://raw.githubusercontent.com/FETS-AI/Front-End/master/src/applications ## Training +### Transfer certificates + +If you have a signed certificate from a previous installation, ensure they are copied before trying to train: +```bash +cd ${fets_root_dir}/bin/ +cp -r ${fets_root_dir_old}/bin/OpenFederatedLearning/bin/federations/pki/client ./OpenFederatedLearning/bin/federations/pki +``` + +### Start Training + ```bash ${fets_root_dir}/bin/FeTS_CLI -d /path/to/output/DataForFeTS \ # input data, ensure "final_seg" is present for each subject -c ${collaborator_common_name} \ # common collaborator name created during setup From f79e80c1d9c0aef5485bca519896a92d2f41e776 Mon Sep 17 00:00:00 2001 From: sarthakpati Date: Tue, 2 Mar 2021 12:16:34 -0500 Subject: [PATCH 02/11] better default logging --- src/applications/FeTS_CLI.cxx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/applications/FeTS_CLI.cxx b/src/applications/FeTS_CLI.cxx index 179688d8..83292f8d 100644 --- a/src/applications/FeTS_CLI.cxx +++ b/src/applications/FeTS_CLI.cxx @@ -60,7 +60,8 @@ int main(int argc, char** argv) } else { - loggingDir = dataDir + "/logs"; + loggingDir = cbica::createTemporaryDirectory() + "/logs"; + std::cout << "Using the following directory as logging directory: " << loggingDir << "\n"; cbica::createDir(loggingDir); } From b067cea0c6f33bd61e95f276c593beaa2c440c32 Mon Sep 17 00:00:00 2001 From: sarthakpati Date: Tue, 2 Mar 2021 13:43:39 -0500 Subject: [PATCH 03/11] updated with links --- docs/runningApplication.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/runningApplication.md b/docs/runningApplication.md index 961dde0b..485b5756 100644 --- a/docs/runningApplication.md +++ b/docs/runningApplication.md @@ -145,7 +145,7 @@ wget https://raw.githubusercontent.com/FETS-AI/Front-End/master/src/applications ### Transfer certificates -If you have a signed certificate from a previous installation, ensure they are copied before trying to train: +If you have a [signed certificate from a previous installation](https://fets-ai.github.io/Front-End/setup#set-up-the-collaborator), ensure they are copied **before** trying to train: ```bash cd ${fets_root_dir}/bin/ cp -r ${fets_root_dir_old}/bin/OpenFederatedLearning/bin/federations/pki/client ./OpenFederatedLearning/bin/federations/pki From 2e041a8f5f34d9f6ab8849ec5f375038b9b377e5 Mon Sep 17 00:00:00 2001 From: sarthakpati Date: Tue, 2 Mar 2021 17:26:00 -0500 Subject: [PATCH 04/11] updated hash for new branch --- OpenFederatedLearning | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OpenFederatedLearning b/OpenFederatedLearning index 7e888f14..d8c2bc67 160000 --- a/OpenFederatedLearning +++ b/OpenFederatedLearning @@ -1 +1 @@ -Subproject commit 7e888f14795de2ac7d210f59902f8c241d7d7b98 +Subproject commit d8c2bc6725e7865d958120551b833cc6f022c734 From 128e81e88696a3e09c0713909223a9b70ea9c46e Mon Sep 17 00:00:00 2001 From: sarthakpati Date: Tue, 2 Mar 2021 17:30:28 -0500 Subject: [PATCH 05/11] removed a redundant variable --- src/applications/FeTS_CLI.cxx | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/applications/FeTS_CLI.cxx b/src/applications/FeTS_CLI.cxx index 83292f8d..0940f45f 100644 --- a/src/applications/FeTS_CLI.cxx +++ b/src/applications/FeTS_CLI.cxx @@ -399,11 +399,7 @@ int main(int argc, char** argv) } // end of trainingRequested check else // for training { - std::string specialArgs, args, hardcodedModelName; - if (trainingRequested) - { - specialArgs = "-col " + colName; - } + std::string args, hardcodedModelName; if (!cbica::isFile(hardcodedPythonPath)) { @@ -416,13 +412,14 @@ int main(int argc, char** argv) args += " -p " + hardcodedPlanName + ".yaml" + " -d " + dataDir - + " -ld " + loggingDir; + + " -ld " + loggingDir + + " -col " + colName; args += device_arg; std::cout << "Starting training...\n"; - if (runCollaboratorTraining(fullCommandToRun + " " + args + " " + specialArgs) != 0) + if (runCollaboratorTraining(fullCommandToRun + " " + args) != 0) { std::cerr << "Couldn't complete the training task, please email admin@fets.ai\n"; return EXIT_FAILURE; From 0f1085e1e786e4b307bb1b2a0ab900721fd0c1f7 Mon Sep 17 00:00:00 2001 From: sarthakpati Date: Tue, 2 Mar 2021 17:46:45 -0500 Subject: [PATCH 06/11] hash updated --- OpenFederatedLearning | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OpenFederatedLearning b/OpenFederatedLearning index d8c2bc67..96ab7300 160000 --- a/OpenFederatedLearning +++ b/OpenFederatedLearning @@ -1 +1 @@ -Subproject commit d8c2bc6725e7865d958120551b833cc6f022c734 +Subproject commit 96ab7300f155091f63ddb6f37bf5beaabf80b040 From 3df4407a073daca1bcdfdb123145b433e4f722bf Mon Sep 17 00:00:00 2001 From: sarthakpati Date: Tue, 2 Mar 2021 17:46:57 -0500 Subject: [PATCH 07/11] cwl update --- data/cwlFiles/FeTS_CLI.cwl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data/cwlFiles/FeTS_CLI.cwl b/data/cwlFiles/FeTS_CLI.cwl index 362597e8..f116434f 100644 --- a/data/cwlFiles/FeTS_CLI.cwl +++ b/data/cwlFiles/FeTS_CLI.cwl @@ -30,6 +30,13 @@ inputs: position: 1 prefix: -cwl doc: Generates a .cwl file for the software. + trainPlan: + type: boolean? + label: YAML file + inputBinding: + position: 1 + prefix: -tp + doc: Training plan.Defaults to 'fets_phase1_1'. LoggingDir: type: Directory? label: Dir with write access From 877a2bd8a9f5c665709ac55ea51f70bd9236fe21 Mon Sep 17 00:00:00 2001 From: sarthakpati Date: Tue, 2 Mar 2021 17:47:09 -0500 Subject: [PATCH 08/11] added the brats 20 model validation --- src/applications/FeTS_CLI.cxx | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/applications/FeTS_CLI.cxx b/src/applications/FeTS_CLI.cxx index 0940f45f..5b58a8a7 100644 --- a/src/applications/FeTS_CLI.cxx +++ b/src/applications/FeTS_CLI.cxx @@ -399,27 +399,38 @@ int main(int argc, char** argv) } // end of trainingRequested check else // for training { - std::string args, hardcodedModelName; - + std::string args = " -d " + dataDir + " -ld " + loggingDir + " -col " + colName + device_arg, + hardcodedModelName; + if (!cbica::isFile(hardcodedPythonPath)) { std::cerr << "The python virtual environment was not found, please refer to documentation to initialize it.\n"; return EXIT_FAILURE; } + + { + std::cout << "Starting model validation of 3DResUNet trained on BraTS20 training data...\n"; + + // brats20 model validation + std::string fullCommandToRun = hardcodedPythonPath + " " + fetsApplicationPath; + fullCommandToRun += "/OpenFederatedLearning/bin/run_fets_validation.py"; + + auto temp_args = args + " -p fets_phase1_validate_full_brats_trained_model_1.yaml"; + + if (std::system((fullCommandToRun + " " + temp_args).c_str()) != 0) + { + std::cerr << "Couldn't complete the BraTS20 model validation task, please email admin@fets.ai\n"; + } + } std::string fullCommandToRun = hardcodedPythonPath + " " + fetsApplicationPath; fullCommandToRun += "/OpenFederatedLearning/bin/run_collaborator_from_flplan.py"; - args += " -p " + hardcodedPlanName + ".yaml" - + " -d " + dataDir - + " -ld " + loggingDir - + " -col " + colName; - - args += device_arg; + auto temp_args = args + " -p " + hardcodedPlanName + ".yaml"; std::cout << "Starting training...\n"; - if (runCollaboratorTraining(fullCommandToRun + " " + args) != 0) + if (runCollaboratorTraining(fullCommandToRun + " " + temp_args) != 0) { std::cerr << "Couldn't complete the training task, please email admin@fets.ai\n"; return EXIT_FAILURE; From ddfb0bc767d724cdd88076c3f8caac725c341587 Mon Sep 17 00:00:00 2001 From: sarthakpati Date: Tue, 2 Mar 2021 17:48:09 -0500 Subject: [PATCH 09/11] version updated for release --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9f0dda25..3b284ab8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ SET( ${PROJECT_NAME}_Variant "Full" ) # the particular variant of CaPTk (Full/Ne SET( PROJECT_VERSION_MAJOR 0 ) SET( PROJECT_VERSION_MINOR 0 ) -SET( PROJECT_VERSION_PATCH 3-dev ) +SET( PROJECT_VERSION_PATCH 3 ) SET( PROJECT_VERSION_TWEAK ) # check for the string "nonRelease" in the PROJECT_VERSION_PATCH variable From 0c5bdc70d68390873cdf4f394e38a82158ac8997 Mon Sep 17 00:00:00 2001 From: sarthakpati Date: Tue, 2 Mar 2021 17:50:56 -0500 Subject: [PATCH 10/11] brats pretrained model download added --- CMakeLists.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b284ab8..613dc260 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -596,18 +596,18 @@ IF( NOT BUILD_DOCUMENTATION_ONLY ) ENDIF() ENDIF() - # SET( DOWNLOAD_LINK "ftp://www.nitrc.org/home/groups/captk/downloads/models/torch/pt_3dresunet_fets_phase1_brats20pretrained_init_1.pbuf" ) - # SET( FILE_TO_EXTRACT "${PROJECT_SOURCE_DIR}/OpenFederatedLearning/bin/federations/weights/pt_3dresunet_fets_phase1_brats20pretrained_init_1.pbuf") + SET( DOWNLOAD_LINK "ftp://www.nitrc.org/home/groups/captk/downloads/models/torch/pt_3dresunet_fets_phase1_brats20pretrained_init_1.pbuf" ) + SET( FILE_TO_EXTRACT "${PROJECT_SOURCE_DIR}/OpenFederatedLearning/bin/federations/weights/pt_3dresunet_fets_phase1_brats20pretrained_init_1.pbuf") - # IF( NOT EXISTS "${FILE_TO_EXTRACT}" ) + IF( NOT EXISTS "${FILE_TO_EXTRACT}" ) - # # download exe from url - # MESSAGE( STATUS "Downloading weights for BraTS evaluation model" ) - # FILE(DOWNLOAD "${DOWNLOAD_LINK}" "${FILE_TO_EXTRACT}" TIMEOUT 1000000 STATUS STATUS_CODE SHOW_PROGRESS) - # IF(NOT STATUS_CODE EQUAL 0) - # MESSAGE(FATAL_ERROR "Failed to download BraTS evaluation model weights. Status=${STATUS_CODE}") - # ENDIF() - # ENDIF() + # download exe from url + MESSAGE( STATUS "Downloading weights for BraTS evaluation model" ) + FILE(DOWNLOAD "${DOWNLOAD_LINK}" "${FILE_TO_EXTRACT}" TIMEOUT 1000000 STATUS STATUS_CODE SHOW_PROGRESS) + IF(NOT STATUS_CODE EQUAL 0) + MESSAGE(FATAL_ERROR "Failed to download BraTS evaluation model weights. Status=${STATUS_CODE}") + ENDIF() + ENDIF() # copy network information ## todo: change to a direct installation so that git doesn't get confused that submodule has changed From e6a4402ca7a3606eb318fd062f72a5893c1ee124 Mon Sep 17 00:00:00 2001 From: sarthakpati Date: Wed, 3 Mar 2021 06:56:01 -0500 Subject: [PATCH 11/11] added sanity checking for sanity check in training --- docs/runningApplication.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/runningApplication.md b/docs/runningApplication.md index 485b5756..04502427 100644 --- a/docs/runningApplication.md +++ b/docs/runningApplication.md @@ -143,7 +143,10 @@ wget https://raw.githubusercontent.com/FETS-AI/Front-End/master/src/applications ## Training -### Transfer certificates +### Ensure Sanity Checking Is Done + +Proceed to training once [sanity check](#sanity-check) is successfully finished. +### Transfer Certificates If you have a [signed certificate from a previous installation](https://fets-ai.github.io/Front-End/setup#set-up-the-collaborator), ensure they are copied **before** trying to train: ```bash