Merge branch 'master' into trvl-mask-layers

.github/workflows/tests.yml

@@ -60,7 +60,7 @@ jobs:
         export CONDA_BUILD_SYSROOT=$CONDA_PREFIX/MacOSX10.9.sdk
         conda build -q conda-recipe
     - name: Upload conda package to NNPDF server
-      if: github.ref == 'refs/heads/master'
+      if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')}}
       shell: bash -l {0}
       run: |
         KEY=$( mktemp )

conda-recipe/meta.yaml

@@ -52,7 +52,7 @@ requirements:
         - sphinxcontrib-bibtex
         - curio >=1.0
         - pineappl >=0.6.2
-        - eko >=0.14.0
+        - eko >=0.14.1
         - fiatlux
         - frozendict  # needed for caching of data loading
 

doc/sphinx/source/releases.rst

@@ -7,11 +7,10 @@ We follow a rolling development model where the tip of the `master branch
 and correct. Binary packages for the latest commit on the branch, with
 appropriate version information are :ref:`generated automatically<CI>` and can
 be :ref:`readily installed<conda>`. In general the version of the code should be
-preferred for producing new results, but see the :ref:`compatibility
-policy<compatibility policy>` below. The main results, such as NNPDF 4.0
-:cite:p:`nnpdf40` will be produced with a frozen :ref:`tag <tags>`, a
-:ref:`conda environment <conda>` and a :ref:`docker image <docker>` so that they
-can be :ref:`reproduced <reproduce40>` entirely.
+preferred for producing new results, but see the compatibility policy below. The
+main results, such as NNPDF 4.0 :cite:p:`nnpdf40` will be produced with a frozen
+:ref:`tag <tags>`, a :ref:`conda environment <conda>` and a :ref:`docker image
+<docker>` so that they can be :ref:`reproduced <reproduce40>` entirely.
 
 .. _tags:
 Tags
@@ -21,9 +20,12 @@ The code is tagged to contextualize the versioning, mark significant
 developments and to mark versions used to produce main results. The
 significant releases since the code was made public are:
 
+`Version 4.0.7 <https://github.com/NNPDF/nnpdf/releases/tag/4.0.7>`_
+    Intermediate release with miscelanous improvements in preparation for 4.0.8.
+    Development release with experimental features
 `Version 4.0.6 <https://github.com/NNPDF/nnpdf/releases/tag/4.0.6>`_
-    The last version that uses C++ objects in n3fit and validphys, and apfel for 
-    PDF evolution. 
+    The last version that uses C++ objects in n3fit and validphys, and apfel for
+    PDF evolution.
 `Version 4.0.5 <https://github.com/NNPDF/nnpdf/releases/tag/4.0.5>`_
     The last version to support legacy genetic algorithms fits based on C++.
 `Version 4.0.4 <https://github.com/NNPDF/nnpdf/releases/tag/4.0.4>`_
@@ -80,4 +82,3 @@ maintained and kept in working order with newer updates, it is highly
 suggested to :ref:`contribute it to the main repository <rules>`,
 along with appropriate tests and documentation. Otherwise you are on your
 own.
-

doc/sphinx/source/vp/theorycov/examples.rst

@@ -154,38 +154,6 @@ a comprehensive set of plots and tables describing the covariance matrices.
       {@plot_diag_cov_comparison@}
    {@endwith@}
 
-   Experimental $\chi^2$
-   ---------------------
-   {@with default_theory@}
-      {@total_experiments_chi2@}
-
-   Total (exp. + th.) $\chi^2$
-   ---------------------------
-      {@chi2_impact_custom@}
-
-   Experimental $\chi^2$ by dataset
-   --------------------------------
-      {@experiments_chi2_table@}
-
-   Total (exp. + th.) $\chi^2$ by dataset
-   --------------------------------------
-      {@experiments_chi2_table_theory@}
-
-   $\chi^2$ including only diagonal theory elements
-   ------------------------------------------------
-      {@chi2_diag_only@}
-
-   Impact of theory covariance matrix on $\chi^2$s 
-   -----------------------------------------------
-      {@plot_datasets_chi2_theory@}
-   {@endwith@}
-
-   Scale variations as a function of the kinematics
-   ------------------------------------------------
-   {@with matched_datasets_from_dataspecs@}
-      [Plots for {@dataset_name@}]({@dataset_report report@})
-   {@endwith@}
-
 
 Validation report
 ----------------- 

n3fit/src/n3fit/backends/keras_backend/MetaModel.py

@@ -399,12 +399,6 @@ def split_replicas(self):
         for i_replica in range(self.num_replicas):
             replica = self.single_replica_generator()
             replica.set_replica_weights(self.get_replica_weights(i_replica))
-
-            # pick single photon
-            if "add_photons" in self.layers:
-                replica.get_layer("add_photons").set_photon(
-                    self.get_layer("add_photons").get_photon(i_replica)
-                )
             replicas.append(replica)
 
         return replicas

n3fit/src/n3fit/checks.py

@@ -394,7 +394,7 @@ def check_deprecated_options(fitting):
 
 
 @make_argcheck
-def check_fiatlux_pdfs_id(replicas, fiatlux, replica_path):
+def check_fiatlux_pdfs_id(replicas, fiatlux):
     if fiatlux is not None:
         luxset = fiatlux["luxset"]
         pdfs_ids = luxset.get_members() - 1  # get_members counts also replica0
@@ -403,3 +403,11 @@ def check_fiatlux_pdfs_id(replicas, fiatlux, replica_path):
             raise CheckError(
                 f"Cannot generate a photon replica with id larger than the number of replicas of the PDFs set {luxset.name}:\nreplica id={max_id}, replicas of {luxset.name} = {pdfs_ids}"
             )
+
+@make_argcheck
+def check_multireplica_qed(replicas, fiatlux):
+    if fiatlux is not None:
+        if len(replicas) > 1:
+            raise CheckError(
+                "At the moment, running a multireplica QED fits is not allowed."
+            )

n3fit/src/n3fit/model_gen.py

@@ -427,20 +427,20 @@ def generate_pdf_model(
         "impose_sumrule": impose_sumrule,
         "scaler": scaler,
     }
-    if photons is not None:
-        if num_replicas > 1:
-            raise ValueError("Photon PDFs are only supported for single replica models. ")
-        single_photon = Photon(photons.theoryid, photons.lux_params, replicas=[1])
-    else:
-        single_photon = None
 
     pdf_model = pdfNN_layer_generator(
         **joint_args, seed=seed, num_replicas=num_replicas, photons=photons
     )
 
+    # Note that the photons are passed unchanged to the single replica generator
+    # computing the photon requires running fiatlux which takes 30' per replica
+    # and so at the moment parallel photons are disabled with a check in checks.py
+    # In order to enable it `single_replica_generator` must take the index of the replica
+    # to select the appropiate photon as all of them will be computed and fixed before the fit
+
     # this is necessary to be able to convert back to single replica models after training
     single_replica_generator = lambda: pdfNN_layer_generator(
-        **joint_args, seed=0, num_replicas=1, photons=single_photon, replica_axis=False
+        **joint_args, seed=0, num_replicas=1, photons=photons, replica_axis=False
     )
     pdf_model.single_replica_generator = single_replica_generator
 

n3fit/src/n3fit/model_trainer.py

@@ -843,6 +843,7 @@ def hyperparametrizable(self, params):
 
         # Generate the grid in x, note this is the same for all partitions
         xinput = self._xgrid_generation()
+
         # Initialize all photon classes for the different replicas:
         if self.lux_params:
             photons = Photon(

n3fit/src/n3fit/performfit.py

@@ -17,6 +17,7 @@
 # Action to be called by validphys
 # All information defining the NN should come here in the "parameters" dict
 @n3fit.checks.can_run_multiple_replicas
+@n3fit.checks.check_multireplica_qed
 @n3fit.checks.check_fiatlux_pdfs_id
 def performfit(
     *,

validphys2/examples/theory_covariance/template_matrix_plots.md

@@ -17,36 +17,4 @@ Diagonal elements of covariance matrices
 ----------------------------------------
 {@with default_theory@}
    {@plot_diag_cov_comparison@}
-{@endwith@}
-
-Experimental $\chi^2$
----------------------
-{@with default_theory@}
-   {@total_chi2_per_point_data@}
-
-Total (exp. + th.) $\chi^2$
----------------------------
-   {@chi2_impact_custom@}
-
-Experimental $\chi^2$ by dataset
---------------------------------
-   {@procs_chi2_table@}
-
-Total (exp. + th.) $\chi^2$ by dataset
---------------------------------------
-   {@procs_chi2_table_theory@}
-
-$\chi^2$ including only diagonal theory elements
-------------------------------------------------
-   {@chi2_diag_only@}
-
-Impact of theory covariance matrix on $\chi^2$s
------------------------------------------------
-   {@plot_datasets_chi2_theory@}
-{@endwith@}
-
-Scale variations as a function of the kinematics
-------------------------------------------------
-{@with matched_datasets_from_dataspecs@}
-   [Plots for {@dataset_name@}]({@dataset_report report@})
-{@endwith@}
+{@endwith@}

validphys2/src/validphys/config.py

@@ -346,6 +346,11 @@ def produce_multiclosure_underlyinglaw(self, fits):
             )
         return self.parse_pdf(laws.pop())
 
+    def produce_fitq0fromfit(self, fitinputcontext):
+        """Given a fit, return the fitting scale according to the theory"""
+        theory = fitinputcontext["theoryid"]
+        return theory.get_description()["Q0"]
+
     def produce_basisfromfit(self, fit):
         """Set the basis from fit config. In the fit config file the basis
         is set using the key ``fitbasis``, but it is exposed to validphys
@@ -906,20 +911,16 @@ def produce_theory_database(self):
         """Produces path to the theory.db file"""
         return self.loader.theorydb_file
 
-    def produce_combined_shift_and_theory_dataspecs(self, theoryconfig, shiftconfig):
-        total_dataspecs = theoryconfig["dataspecs"] + shiftconfig["dataspecs"]
-        matched_datasets = self.produce_matched_datasets_from_dataspecs(total_dataspecs)
+    def produce_combined_shift_and_theory_dataspecs(self, dataspecs):
+        matched_datasets = self.produce_matched_datasets_from_dataspecs(dataspecs)
         for ns in matched_datasets:
             ns["dataspecs"] = self.produce_dataspecs_with_matched_cuts(ns["dataspecs"])
-        new_theoryconfig = []
-        new_shiftconfig = []
-        len_th = len(theoryconfig["dataspecs"])
+        new_dataspecs = []
+        len_th = len(dataspecs)
         for s in matched_datasets:
-            new_theoryconfig.append(ChainMap({"dataspecs": s["dataspecs"][:len_th]}, s))
-            new_shiftconfig.append(ChainMap({"dataspecs": s["dataspecs"][len_th:]}, s))
+            new_dataspecs.append(ChainMap({"dataspecs": s["dataspecs"][len_th:]}, s))
         return {
-            "shiftconfig": {"dataspecs": new_shiftconfig, "original": shiftconfig},
-            "theoryconfig": {"dataspecs": new_theoryconfig, "original": theoryconfig},
+            "dataspecs": {"dataspecs": new_dataspecs, "original": dataspecs },
         }
 
     # TODO: Worth it to do some black magic to not pass params explicitly?

validphys2/src/validphys/eff_exponents.py

@@ -346,6 +346,7 @@ def previous_effective_exponents_table(fit: FitSpec):
 def next_effective_exponents_table(
     pdf: PDF,
     *,
+    fitq0fromfit: (numbers.Real, type(None)) = None,
     x1_alpha: numbers.Real = 1e-6,
     x2_alpha: numbers.Real = 1e-3,
     x1_beta: numbers.Real = 0.65,
@@ -374,7 +375,12 @@ def next_effective_exponents_table(
         max(2x68% c.l. upper value evaluated at x=`x1_beta` and x=`x2_beta`)
 
     """
-    Qmin = pdf.q_min
+    if fitq0fromfit is None:
+        log.warning("Computing the next effective exponent directly from the PDF")
+        Qmin = pdf.q_min
+        log.warning(f"Taking q = {Qmin} GeV as the reference scale")
+    else:
+        Qmin = fitq0fromfit
 
     alpha_effs = alpha_eff(
         pdf, xmin=x1_alpha, xmax=x2_alpha, npoints=2, Q=Qmin, basis=basis, flavours=flavours