Skip to content

Commit

Permalink
Merge pull request #48 from mach3-software/feature_Expand
Browse files Browse the repository at this point in the history
Expand (again) Tutorial
  • Loading branch information
KSkwarczynski authored Nov 25, 2024
2 parents e9501ba + 2cd2e07 commit 1a1bb85
Show file tree
Hide file tree
Showing 19 changed files with 263 additions and 108 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/CIPythonValidations.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,5 +39,5 @@ jobs:
export PYTHONPATH=$PYTHONPATH:$PWD/python-test-modules/
export MACH3=$PWD
python3 -m pytest --config Inputs/ManagerTest.yaml CIValidations/PythonTests
python3 -m pytest --config Inputs/FitterConfig.yaml CIValidations/PythonTests
85 changes: 13 additions & 72 deletions .github/workflows/CIValidations.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,48 +22,24 @@ jobs:
matrix:
include:
- name: Reweight Validations
test_1: ./CIValidations/SplineValidations
test_2: ./CIValidations/SamplePDFValidations
test_3: ./CIValidations/NuOscillatorInterfaceValidations
test_4: empty
test_5: empty
test_6: empty
test_7: empty
test_8: empty
test_9: empty
test_1: MaCh3CLI --ReweightValidations
test_2: empty
test_3: empty

- name: Covariance Validations
test_1: ./CIValidations/CovarianceValidations
test_2: ./CIValidations/MaCh3ModeValidations
test_3: ./CIValidations/UnitTests/manager_tests
test_4: empty
test_5: empty
test_6: empty
test_7: empty
test_8: empty
test_9: empty
test_1: MaCh3CLI --CovarianceValidations
test_2: empty
test_3: empty

- name: Fitter Validations
test_1: ./CIValidations/FitterValidations
test_2: ./bin/ProcessMCMC bin/TutorialDiagConfig.yaml MCMC_Test.root
test_3: ./bin/DiagMCMC MCMC_Test.root bin/TutorialDiagConfig.yaml
test_4: ./bin/RHat 10 MCMC_Test.root MCMC_Test.root MCMC_Test.root MCMC_Test.root
test_5: ./bin/CombineMaCh3Chains -o MergedChain.root MCMC_Test.root MCMC_Test.root MCMC_Test.root MCMC_Test.root
test_6: ./bin/GetPenaltyTerm MCMC_Test.root bin/TutorialDiagConfig.yaml
test_7: ./bin/MatrixPlotter bin/TutorialDiagConfig.yaml MCMC_Test_drawCorr.root
test_8: bin/GetPostfitParamPlots MCMC_Test_drawCorr.root
test_9: bin/PlotLLH MCMC_Test.root
test_1: MaCh3CLI --FitterValidations
test_2: empty
test_3: empty

- name: NuMCMC Tools Validations
test_1: ./CIValidations/NuMCMCvalidations.sh
test_1: MaCh3CLI --NuMCMCToolsValidations
test_2: empty
test_3: empty
test_4: empty
test_5: empty
test_6: empty
test_7: empty
test_8: empty
test_9: empty

container:
image: ghcr.io/mach3-software/mach3:alma9latest
Expand All @@ -90,10 +66,11 @@ jobs:
run: |
source /opt/MaCh3Validations/build/bin/setup.MaCh3.sh
source /opt/MaCh3Validations/build/bin/setup.MaCh3Tutorial.sh
cd /opt/MaCh3Validations/build
cd /opt/MaCh3Validations/build/CIValidations
${{ matrix.test_1 }}
#KS: GHA is stupi when it comes to arrays. I tried something fancy but it was crashing terribly
#KS: GHA is stupid when it comes to arrays. I tried something fancy but it was crashing terribly
#If you know how to write this better please let me know
# Run the second test if it is defined
Expand All @@ -108,39 +85,3 @@ jobs:
echo "Performing Third test"
${{ matrix.test_3 }}
fi
if [[ "${{ matrix.test_4 }}" != "empty" ]]; then
echo " "
echo "Performing test 4"
${{ matrix.test_4 }}
fi
if [[ "${{ matrix.test_5 }}" != "empty" ]]; then
echo " "
echo "Performing test 5"
${{ matrix.test_5 }}
fi
if [[ "${{ matrix.test_6 }}" != "empty" ]]; then
echo " "
echo "Performing test 6"
${{ matrix.test_6 }}
fi
if [[ "${{ matrix.test_7 }}" != "empty" ]]; then
echo " "
echo "Performing test 7"
${{ matrix.test_7 }}
fi
if [[ "${{ matrix.test_8 }}" != "empty" ]]; then
echo " "
echo "Performing test 8"
${{ matrix.test_8 }}
fi
if [[ "${{ matrix.test_9 }}" != "empty" ]]; then
echo " "
echo "Performing test 9"
${{ matrix.test_9 }}
fi
3 changes: 2 additions & 1 deletion CIValidations/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ endforeach(app)
install(PROGRAMS
NuMCMCvalidations.sh
TestNuMCMC.py
MaCh3CLI

DESTINATION CIValidations)

Expand All @@ -29,4 +30,4 @@ install(DIRECTORY

if(MaCh3Tutorial_UNITTESTS_ENABLED)
add_subdirectory(UnitTests)
endif()
endif()
2 changes: 1 addition & 1 deletion CIValidations/CovarianceValidations.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ int main(int argc, char *argv[])
}

////////////// Normal Xsec //////////////
std::vector<std::string> xsecCovMatrixFile = {"Inputs/SystematicsTest.yaml"};
std::vector<std::string> xsecCovMatrixFile = {"Inputs/SystematicModel.yaml"};
covarianceXsec* xsec = new covarianceXsec(xsecCovMatrixFile, "xsec_cov");

std::vector<double> ParProp = {1.05, 0.90, 1.10, 1.05, 0.25, 1.70, 3.20, -1.10, -1.70};
Expand Down
7 changes: 2 additions & 5 deletions CIValidations/FitterValidations.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,6 @@
#include "Utils/Comparison.h"
#include "samplePDF/samplePDFTutorial.h"

/// @todo add sample PDF object!
///
///
/// Current tests include
/// Running MCMC
/// Running MinuitFit
Expand All @@ -15,7 +12,7 @@
/// Running GetStepScaleBasedOnLLHScan
void FitVal(const std::string& Algo, bool MoreTests)
{
std::string ManagerInput = "Inputs/ManagerTest.yaml";
std::string ManagerInput = "Inputs/FitterConfig.yaml";
manager *FitManager = new manager(ManagerInput);

MACH3LOG_INFO("Testing {}", Algo);
Expand Down Expand Up @@ -73,7 +70,7 @@ void FitVal(const std::string& Algo, bool MoreTests)

void StartFromPosteriorTest(const std::string& PreviousName)
{
std::string ManagerInput = "Inputs/ManagerTest.yaml";
std::string ManagerInput = "Inputs/FitterConfig.yaml";
manager *FitManager = new manager(ManagerInput);

FitManager->OverrideSettings("General", "OutputFile", "MCMC_Test_Start.root");
Expand Down
102 changes: 102 additions & 0 deletions CIValidations/MaCh3CLI
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
#!/bin/bash

# Check if no arguments are provided
if [ "$#" -eq 0 ]; then
echo "[ERROR]: No argument specified, please use --help"
exit 1
fi

# Display help message
show_help() {
echo "Arguments:"
echo -e "\t --FitterValidations : Integration testing of running MCMC and all plotting scripts"
echo -e "\t --ReweightValidations : Checking if reweighting works"
echo -e "\t --CovarianceValidations : Checking if covariances works"
echo -e "\t --NuMCMCToolsValidations : Checking if interface to NuMCMC Tools works"
echo -e "\t --all : Run all checks"
}

# Run validation functions
run_reweight_validations() {
./CIValidations/SplineValidations
./CIValidations/SamplePDFValidations
./CIValidations/NuOscillatorInterfaceValidations
}

run_fitter_validations() {
./CIValidations/FitterValidations
./bin/ProcessMCMC bin/TutorialDiagConfig.yaml MCMC_Test.root
./bin/DiagMCMC MCMC_Test.root bin/TutorialDiagConfig.yaml
./bin/RHat 10 MCMC_Test.root MCMC_Test.root MCMC_Test.root MCMC_Test.root
./bin/CombineMaCh3Chains -o MergedChain.root MCMC_Test.root MCMC_Test.root MCMC_Test.root MCMC_Test.root
./bin/GetPenaltyTerm MCMC_Test.root bin/TutorialDiagConfig.yaml
./bin/MatrixPlotter bin/TutorialDiagConfig.yaml MCMC_Test_drawCorr.root
./bin/GetPostfitParamPlots MCMC_Test_drawCorr.root
./bin/PlotLLH MCMC_Test.root
}

run_covariance_validations() {
./CIValidations/CovarianceValidations
./CIValidations/MaCh3ModeValidations
./CIValidations/UnitTests/manager_tests
}

run_nu_mcmc_tools_validations() {
./bin/MCMCTutorial Inputs/FitterConfig.yaml
./CIValidations/NuMCMCconversion Test.root NewChain.root
pip install -r NuMCMCTools/requirements.txt
export PYTHONPATH="$PYTHONPATH:${MaCh3Tutorial_ROOT}/NuMCMCTools"
cp NewChain.root CIValidations/NewChain.root
python3 CIValidations/TestNuMCMC.py
}

# Parse command-line arguments
while [[ $# -gt 0 ]]; do
case $1 in
-?|--help)
show_help
exit 0
;;
--ReweightValidations)
RUN_REWEIGHT=true
;;
--FitterValidations)
RUN_FITTER=true
;;
--CovarianceValidations)
RUN_COVARIANCE=true
;;
--NuMCMCToolsValidations)
RUN_NU_MCMC=true
;;
--all)
RUN_ALL=true
;;
*)
echo "[ERROR]: Unknown argument '$1'. Use --help for more information."
exit 1
;;
esac
shift
done

# Navigate to the MaCh3Tutorial root directory
cd "${MaCh3Tutorial_ROOT}" || {
echo "[ERROR]: Could not navigate to MaCh3Tutorial_ROOT directory."
exit 1
}

# Execute validations based on the arguments
if [ "$RUN_ALL" = true ]; then
run_reweight_validations
run_fitter_validations
run_covariance_validations
run_nu_mcmc_tools_validations
else
[ "$RUN_REWEIGHT" = true ] && run_reweight_validations
[ "$RUN_FITTER" = true ] && run_fitter_validations
[ "$RUN_COVARIANCE" = true ] && run_covariance_validations
[ "$RUN_NU_MCMC" = true ] && run_nu_mcmc_tools_validations
fi

echo "Validation process completed!"
2 changes: 1 addition & 1 deletion CIValidations/NuMCMCvalidations.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
./bin/MCMCTutorial Inputs/ManagerTest.yaml
./bin/MCMCTutorial Inputs/FitterConfig.yaml

./CIValidations/NuMCMCconversion Test.root NewChain.root

Expand Down
6 changes: 3 additions & 3 deletions CIValidations/PythonTests/test_fitter_sample_pdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@ def __init__(self, xsec_cov: covariance.CovarianceXsec):

self.proposed_param_array = np.asarray(xsec_cov.get_proposal_array())

self.setup_splines("Inputs/SplineFile.root")
self.setup_splines("Inputs/MC/SplineFile.root")

self.setup_MC("Inputs/SplineFile.root")
self.setup_MC("Inputs/MC/SplineFile.root")

# Get the nominal event rate which we will later use as our "data"
self.setup_asimov_data()
Expand Down Expand Up @@ -213,4 +213,4 @@ def test_sample_pdf(pytestconfig):

# RUUUUUN!!!
mcmc_fitter.set_chain_length(1000)
mcmc_fitter.run()
mcmc_fitter.run()
2 changes: 1 addition & 1 deletion CIValidations/SamplePDFValidations.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ int main(int argc, char *argv[])
throw MaCh3Exception(__FILE__ , __LINE__ );
}

std::vector<std::string> xsecCovMatrixFile = {"Inputs/SystematicsTest.yaml"};
std::vector<std::string> xsecCovMatrixFile = {"Inputs/SystematicModel.yaml"};
covarianceXsec* xsec = new covarianceXsec(xsecCovMatrixFile, "xsec_cov");

std::vector<std::string> OscCovMatrixFile = {"Inputs/Osc_Test.yaml"};
Expand Down
2 changes: 1 addition & 1 deletion CIValidations/SplineValidations.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ int main(int argc, char *argv[])
}
MACH3LOG_INFO("Testing Spline Monolith");

std::string Input = "Inputs/SplineFile.root";
std::string Input = "Inputs/MC/SplineFile.root";
std::vector<std::string> Dials = {"Spline_0", "Spline_1", "Spline_2", "Spline_3"};
std::vector<double> Dial_Values = {1.21, 1, 1, 1};
std::vector< std::vector<TResponseFunction_red*> > MasterSpline = GetMasterSpline(Input, Dials);
Expand Down
2 changes: 2 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ if(NOT MaCh3_FOUND)
GITHUB_REPOSITORY mach3-software/MaCh3
OPTIONS
"MaCh3_PYTHON_ENABLED ${PYTHON_ENABLED}"
"Prob3ppLinear_ENABLED ON"
"NuFastLinear_ENABLED ON"
)
# MaCh3 sets up cmessage for us, so can't cmessage until we've set up MaCh3
cmessage(STATUS "Didn't find MaCh3, attempting to use built-in MaCh3")
Expand Down
2 changes: 1 addition & 1 deletion Inputs/ManagerTest.yaml → Inputs/FitterConfig.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ General:
Systematics:
XsecCovName: "xsec_cov"
XsecFix: [ ]
XsecCovFile: ["Inputs/SystematicsTest.yaml",
XsecCovFile: ["Inputs/SystematicModel.yaml",
#"Inputs/PCATest.yaml"
]
XsecPCAThreshold: -1
Expand Down
File renamed without changes.
File renamed without changes.
17 changes: 17 additions & 0 deletions Inputs/NuOscillator/Prob3ppLinear.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
General:
Verbosity: "NONE"
CosineZIgnored: true
CalculationType: "Unbinned"

OscProbCalcerSetup:
ImplementationName: "Prob3ppLinear"
OscChannelMapping:
- Entry: "Electron:Electron"
- Entry: "Electron:Muon"
- Entry: "Electron:Tau"
- Entry: "Muon:Electron"
- Entry: "Muon:Muon"
- Entry: "Muon:Tau"
- Entry: "Tau:Electron"
- Entry: "Tau:Muon"
- Entry: "Tau:Tau"
2 changes: 1 addition & 1 deletion Inputs/SamplePDF_Tutorial.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Binning:
XVarBins: [0., 0.5, 1., 1.25, 1.5, 1.75, 2., 2.25, 2.5, 2.75, 3., 3.25, 3.5, 3.75, 4., 5., 6., 10.]

InputFiles:
mtupleprefix: "Inputs/NuWro"
mtupleprefix: "Inputs/MC/NuWro"
mtuplesuffix: "_FlatTree.root"
# No Splines
splineprefix: "fixme"
Expand Down
File renamed without changes.
Loading

0 comments on commit 1a1bb85

Please sign in to comment.