Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Hyperopt loss #1726

Merged
merged 20 commits into from
Mar 7, 2024
Merged
Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Remove phi2 from names
Cmurilochem committed Mar 7, 2024
commit 7a06180b37b2d756c64263dbb53e5487f9427ecd
32 changes: 16 additions & 16 deletions n3fit/src/n3fit/hyper_optimization/rewards.py
Cmurilochem marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -85,7 +85,7 @@ def __init__(
self.reduce_over_replicas = self._parse_statistic(replica_statistic, "replica_statistic")
self.reduce_over_folds = self._parse_statistic(fold_statistic, "fold_statistic")

self.phi2_vector = []
self.phi_vector = []
self.chi2_matrix = []

self.penalties = {}
@@ -136,59 +136,59 @@ def compute_loss(
>>> pdf_model = generate_pdf_model(nodes=[8], activations=['linear'], seed=0, num_replicas=2, flav_info=fake_fl, fitbasis="FLAVOUR")
>>> loss = hyper.compute_loss(penalties, experimental_loss, pdf_model, experimental_data)
"""
# calculate phi2 for a given k-fold using vpinterface and validphys
phi2_per_fold = compute_phi(N3PDF(pdf_model.split_replicas()), experimental_data)
# calculate phi for a given k-fold using vpinterface and validphys
phi_per_fold = compute_phi(N3PDF(pdf_model.split_replicas()), experimental_data)

# update hyperopt metrics
# these are saved in the phi2_vector and chi2_matrix attributes, excluding penalties
self._save_hyperopt_metrics(phi2_per_fold, experimental_loss, penalties, fold_idx)
# these are saved in the phi_vector and chi2_matrix attributes, excluding penalties
self._save_hyperopt_metrics(phi_per_fold, experimental_loss, penalties, fold_idx)

# include penalties to experimental loss
# this allows introduction of statistics also to penalties
experimental_loss_w_penalties = experimental_loss + sum(penalties.values())

# add penalties to phi2 in the form of a sum of per-replicas averages
phi2_per_fold += sum(np.mean(penalty) for penalty in penalties.values())
# add penalties to phi in the form of a sum of per-replicas averages
phi_per_fold += sum(np.mean(penalty) for penalty in penalties.values())

# define loss for hyperopt according to the chosen loss_type
if self.loss_type == "chi2":
# calculate statistics of chi2 over replicas for a given k-fold
loss = self.reduce_over_replicas(experimental_loss_w_penalties)
elif self.loss_type == "phi2":
loss = phi2_per_fold
loss = phi_per_fold**2

return loss

def _save_hyperopt_metrics(
self,
phi2_per_fold: float,
phi_per_fold: float,
chi2_per_fold: np.ndarray,
penalties: Dict[str, np.ndarray],
fold_idx: int = 0,
) -> None:
"""
Save all chi2 and phi2 calculated metrics per replica and per fold, including penalties.
Save all chi2 and phi calculated metrics per replica and per fold, including penalties.
Parameters
----------
phi2_per_fold: float
Computed phi2 for a given k-fold
phi_per_fold: float
Computed phi for a given k-fold
chi2_per_fold: np.ndarray
Computed chi2 for each replica for a given k-fold
penalties: Dict[str, np.ndarray]
dictionary of all penalties with their names
fold_idx: int
k-fold index. Defaults to 0.
"""
# reset chi2 and phi2 arrays for every trial
# reset chi2 and phi arrays for every trial
if fold_idx == 0:
self.phi2_vector = []
self.phi_vector = []
self.chi2_matrix = []
self.penalties = {}

# populate chi2 matrix and phi2 vector calculated for a given k-fold
# populate chi2 matrix and phi vector calculated for a given k-fold
self.chi2_matrix.append(chi2_per_fold)
self.phi2_vector.append(phi2_per_fold)
self.phi_vector.append(phi_per_fold)

# save penalties per replica for a given k-fold
for name, values in penalties.items():
16 changes: 8 additions & 8 deletions n3fit/src/n3fit/model_trainer.py
Original file line number Diff line number Diff line change
@@ -890,8 +890,8 @@ def hyperparametrizable(self, params):
# And lists to save hyperopt utilities
pdfs_per_fold = []
exp_models = []
# phi2 evaluated over training/validation exp data
trvl_phi2_per_fold = []
# phi evaluated over training/validation exp data
trvl_phi_per_fold = []

# Generate the grid in x, note this is the same for all partitions
xinput = self._xgrid_generation()
@@ -999,7 +999,7 @@ def hyperparametrizable(self, params):
for penalty in self.hyper_penalties
}

# Extracting the necessary data to compute phi2
# Extracting the necessary data to compute phi
# First, create a list of `validphys.core.DataGroupSpec`
# containing only exp datasets within the held out fold
experimental_data = self._filter_datagroupspec(partition["datasets"])
@@ -1023,14 +1023,14 @@ def hyperparametrizable(self, params):
exp_name for item in trvl_partitions for exp_name in item['datasets']
]
trvl_data = self._filter_datagroupspec(trvl_exp_names)
# evaluate phi2 on training/validation exp set
trvl_phi2 = compute_phi(N3PDF(pdf_model.split_replicas()), trvl_data)
# evaluate phi on training/validation exp set
trvl_phi = compute_phi(N3PDF(pdf_model.split_replicas()), trvl_data)

# Now save all information from this fold
l_hyper.append(hyper_loss)
l_valid.append(validation_loss)
l_exper.append(experimental_loss)
trvl_phi2_per_fold.append(trvl_phi2)
trvl_phi_per_fold.append(trvl_phi)
pdfs_per_fold.append(pdf_model)
exp_models.append(models["experimental"])

@@ -1066,10 +1066,10 @@ def hyperparametrizable(self, params):
"experimental_loss": np.average(l_exper),
"kfold_meta": {
"validation_losses": l_valid,
"validation_losses_phi2": np.array(trvl_phi2_per_fold),
"validation_losses_phi": np.array(trvl_phi_per_fold),
"experimental_losses": l_exper,
"hyper_losses": np.array(self._hyper_loss.chi2_matrix),
"hyper_losses_phi2": np.array(self._hyper_loss.phi2_vector),
"hyper_losses_phi": np.array(self._hyper_loss.phi_vector),
"penalties": {
name: np.array(values)
Cmurilochem marked this conversation as resolved.
Show resolved Hide resolved
for name, values in self._hyper_loss.penalties.items()