Pineappl integration with validphys

conda-recipe/meta.yaml

@@ -52,6 +52,7 @@ requirements:
         - sphinxcontrib-bibtex
         - docutils =0.16 # This dependency is not explicity needed but https://github.com/NNPDF/nnpdf/issues/1220
         - curio >=1.0
+        - pineappl >=0.5.2
 
 test:
     requires:

n3fit/src/n3fit/layers/observable.py

@@ -5,7 +5,7 @@
 
 
 def _is_unique(list_of_arrays):
-    """ Check whether the list of arrays more than one different arrays """
+    """Check whether the list of arrays more than one different arrays"""
     the_first = list_of_arrays[0]
     for i in list_of_arrays[1:]:
         if not np.array_equal(the_first, i):
@@ -15,40 +15,40 @@ def _is_unique(list_of_arrays):
 
 class Observable(MetaLayer, ABC):
     """
-        This class is the parent of the DIS and DY convolutions.
-        All backend-dependent code necessary for the convolutions
-                                    is (must be) concentrated here
+    This class is the parent of the DIS and DY convolutions.
+    All backend-dependent code necessary for the convolutions
+                                is (must be) concentrated here
 
-        The methods gen_mask and call must be overriden by the observables
-        where
-            - gen_mask: it is called by the initializer and generates the mask between
-                        fktables and pdfs
-            - call: this is what does the actual operation
+    The methods gen_mask and call must be overriden by the observables
+    where
+        - gen_mask: it is called by the initializer and generates the mask between
+                    fktables and pdfs
+        - call: this is what does the actual operation
 
 
-        Parameters
-        ----------
-            fktable_dicts: list
-                list of fktable_dicts which define basis and xgrid for the fktables in the list
-            fktable_arr: list
-                list of fktables for this observable
-            operation_name: str
-                string defining the name of the operation to be applied to the fktables
-            nfl: int
-                number of flavours in the pdf (default:14)
+    Parameters
+    ----------
+        fktable_data: list[validphys.coredata.FKTableData]
+            list of FK which define basis and xgrid for the fktables in the list
+        fktable_arr: list
+            list of fktables for this observable
+        operation_name: str
+            string defining the name of the operation to be applied to the fktables
+        nfl: int
+            number of flavours in the pdf (default:14)
     """
 
-    def __init__(self, fktable_dicts, fktable_arr, operation_name, nfl=14, **kwargs):
+    def __init__(self, fktable_data, fktable_arr, operation_name, nfl=14, **kwargs):
         super(MetaLayer, self).__init__(**kwargs)
 
         self.nfl = nfl
 
         basis = []
         xgrids = []
         self.fktables = []
-        for fktable, fk in zip(fktable_dicts, fktable_arr):
-            xgrids.append(fktable["xgrid"])
-            basis.append(fktable["basis"])
+        for fkdata, fk in zip(fktable_data, fktable_arr):
+            xgrids.append(fkdata.xgrid.reshape(1, -1))
+            basis.append(fkdata.luminosity_mapping)
             self.fktables.append(op.numpy_to_tensor(fk))
 
         # check how many xgrids this dataset needs

n3fit/src/n3fit/model_gen.py

@@ -134,72 +134,72 @@ def observable_generator(
     model_obs_ex = []
     model_inputs = []
     # The first step is to compute the observable for each of the datasets
-    for dataset_dict in spec_dict["datasets"]:
+    for dataset in spec_dict["datasets"]:
         # Get the generic information of the dataset
-        dataset_name = dataset_dict["name"]
+        dataset_name = dataset.name
 
         # Look at what kind of layer do we need for this dataset
-        if dataset_dict["hadronic"]:
+        if dataset.hadronic:
             Obs_Layer = DY
         else:
             Obs_Layer = DIS
 
         # Set the operation (if any) to be applied to the fktables of this dataset
-        operation_name = dataset_dict["operation"]
+        operation_name = dataset.operation
 
         # Now generate the observable layer, which takes the following information:
         # operation name
         # dataset name
-        # list of fktable_dictionaries
+        # list of validphys.coredata.FKTableData objects
         #   these will then be used to check how many different pdf inputs are needed
         #   (and convolutions if given the case)
 
         if spec_dict["positivity"]:
             # Positivity (and integrability, which is a special kind of positivity...)
             # enters only at the "training" part of the models
             obs_layer_tr = Obs_Layer(
-                dataset_dict["fktables"],
-                dataset_dict["tr_fktables"],
+                dataset.fktables_data,
+                dataset.training_fktables(),
                 operation_name,
                 name=f"dat_{dataset_name}",
             )
             obs_layer_ex = obs_layer_vl = None
         elif spec_dict.get("data_transformation_tr") is not None:
             # Data transformation needs access to the full array of output data
             obs_layer_ex = Obs_Layer(
-                dataset_dict["fktables"],
-                dataset_dict["ex_fktables"],
+                dataset.fktables_data,
+                dataset.fktables(),
                 operation_name,
                 name=f"exp_{dataset_name}",
             )
             obs_layer_tr = obs_layer_vl = obs_layer_ex
         else:
             obs_layer_tr = Obs_Layer(
-                dataset_dict["fktables"],
-                dataset_dict["tr_fktables"],
+                dataset.fktables_data,
+                dataset.training_fktables(),
                 operation_name,
                 name=f"dat_{dataset_name}",
             )
             obs_layer_ex = Obs_Layer(
-                dataset_dict["fktables"],
-                dataset_dict["ex_fktables"],
+                dataset.fktables_data,
+                dataset.fktables(),
                 operation_name,
                 name=f"exp_{dataset_name}",
             )
             obs_layer_vl = Obs_Layer(
-                dataset_dict["fktables"],
-                dataset_dict["vl_fktables"],
+                dataset.fktables_data,
+                dataset.validation_fktables(),
                 operation_name,
                 name=f"val_{dataset_name}",
             )
 
         # To know how many xpoints we compute we are duplicating functionality from obs_layer
         if obs_layer_tr.splitting is None:
-            xgrid = dataset_dict["fktables"][0]["xgrid"]
+            xgrid = dataset.fktables_data[0].xgrid.reshape(1, -1)
             model_inputs.append(xgrid)
             dataset_xsizes.append(xgrid.shape[1])
         else:
-            xgrids = [i["xgrid"] for i in dataset_dict["fktables"]]
+            xgrids = [i.xgrid.reshape(1, -1) for i in dataset.fktables_data]
             model_inputs += xgrids
             dataset_xsizes.append(sum([i.shape[1] for i in xgrids]))
 

n3fit/src/n3fit/model_trainer.py

@@ -131,6 +131,8 @@ def __init__(
         """
         # Save all input information
         self.exp_info = exp_info
+        if pos_info is None:
+            pos_info = []
         self.pos_info = pos_info
         self.integ_info = integ_info
         if self.integ_info is not None:
@@ -275,7 +277,7 @@ def _fill_the_dictionaries(self):
             self.experimental["ndata"] += nd_tr + nd_vl
 
             for dataset in exp_dict["datasets"]:
-                self.all_datasets.append(dataset["name"])
+                self.all_datasets.append(dataset.name)
         self.all_datasets = set(self.all_datasets)
 
         for pos_dict in self.pos_info:
@@ -532,7 +534,7 @@ def _generate_observables(
             force_set_smallest = input_arr.min() > 1e-9
             if force_set_smallest:
                 new_xgrid = np.linspace(
-                    start=1/input_arr_size, stop=1.0, endpoint=False, num=input_arr_size
+                    start=1 / input_arr_size, stop=1.0, endpoint=False, num=input_arr_size
                 )
             else:
                 new_xgrid = np.linspace(start=0, stop=1.0, endpoint=False, num=input_arr_size)
@@ -566,8 +568,7 @@ def _generate_observables(
                 scaler = PchipInterpolator(map_from, map_to)
             except ValueError:
                 raise ValueError(
-                    "interpolation_points is larger than the number of unique "
-                                    "input x-values"
+                    "interpolation_points is larger than the number of unique " "input x-values"
                 )
             self._scaler = lambda x: np.concatenate([scaler(np.log(x)), x], axis=-1)
 
@@ -918,7 +919,9 @@ def hyperparametrizable(self, params):
             # by adding it to this dictionary
             dict_out = {
                 "status": passed,
-                "loss": self._hyper_loss(fold_losses=l_hyper, n3pdfs=n3pdfs, experimental_models=exp_models),
+                "loss": self._hyper_loss(
+                    fold_losses=l_hyper, n3pdfs=n3pdfs, experimental_models=exp_models
+                ),
                 "validation_loss": np.average(l_valid),
                 "experimental_loss": np.average(l_exper),
                 "kfold_meta": {