From 8a11bdb4b132f6803485c01c2d222cad8e9d3c66 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Fri, 29 Dec 2023 17:19:19 +0100
Subject: [PATCH 01/10] solve_network: option to inject custom extra
 functionalities from source file

---
 config/config.default.yaml   | 1 +
 doc/configtables/solving.csv | 1 +
 doc/release_notes.rst        | 5 +++++
 rules/solve_myopic.smk       | 3 +++
 rules/solve_overnight.smk    | 3 +++
 rules/solve_perfect.smk      | 3 +++
 scripts/solve_network.py     | 7 +++++++
 7 files changed, 23 insertions(+)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index a6df173ba..0ff742e70 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -627,6 +627,7 @@ solving:
     skip_iterations: true
     rolling_horizon: false
     seed: 123
+    custom_extra_functionality: "data/custom_extra_functionality.py"
     # options that go into the optimize function
     track_iterations: false
     min_iterations: 4
diff --git a/doc/configtables/solving.csv b/doc/configtables/solving.csv
index 45d50d844..9d47c0430 100644
--- a/doc/configtables/solving.csv
+++ b/doc/configtables/solving.csv
@@ -6,6 +6,7 @@ options,,,
 -- skip_iterations,bool,"{'true','false'}","Skip iterating, do not update impedances of branches. Defaults to true."
 -- rolling_horizon,bool,"{'true','false'}","Whether to optimize the network in a rolling horizon manner, where the snapshot range is split into slices of size `horizon` which are solved consecutively."
 -- seed,--,int,Random seed for increased deterministic behaviour.
+-- custom_extra_functionality,--,str,Path to a Python file with custom extra functionality code to be injected into the solving rules of the workflow.
 -- track_iterations,bool,"{'true','false'}",Flag whether to store the intermediate branch capacities and objective function values are recorded for each iteration in ``network.lines['s_nom_opt_X']`` (where ``X`` labels the iteration)
 -- min_iterations,--,int,Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run.
 -- max_iterations,--,int,Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run.
diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index d7931f0ef..a7644682c 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -54,6 +54,11 @@ Upcoming Release
   reconnected to the main Ukrainian grid with the configuration option
   `reconnect_crimea`.
 
+* Add option to reference an additional source file where users can specify
+  custom ``extra_functionality`` constraints in the configuration file. The
+  default setting points to an empty hull at
+  ``data/custom_extra_functionality.py``.
+
 
 **Bugs and Compatibility**
 
diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk
index 8a93d24a8..217547b97 100644
--- a/rules/solve_myopic.smk
+++ b/rules/solve_myopic.smk
@@ -88,6 +88,9 @@ rule solve_sector_network_myopic:
         co2_sequestration_potential=config["sector"].get(
             "co2_sequestration_potential", 200
         ),
+        custom_extra_functionality=workflow.source_path(
+            config["solver"]["options"].get("custom_extra_functionality", "")
+        ),
     input:
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk
index c77007609..8ac56db8c 100644
--- a/rules/solve_overnight.smk
+++ b/rules/solve_overnight.smk
@@ -11,6 +11,9 @@ rule solve_sector_network:
         co2_sequestration_potential=config["sector"].get(
             "co2_sequestration_potential", 200
         ),
+        custom_extra_functionality=workflow.source_path(
+            config["solver"]["options"].get("custom_extra_functionality", "")
+        ),
     input:
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk
index ef4e367d8..322ced8dd 100644
--- a/rules/solve_perfect.smk
+++ b/rules/solve_perfect.smk
@@ -118,6 +118,9 @@ rule solve_sector_network_perfect:
         co2_sequestration_potential=config["sector"].get(
             "co2_sequestration_potential", 200
         ),
+        custom_extra_functionality=workflow.source_path(
+            config["solver"]["options"].get("custom_extra_functionality", "")
+        ),
     input:
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index ff2a2f232..539c4e72e 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -26,7 +26,9 @@
     the workflow for all scenarios in the configuration file (``scenario:``)
     based on the rule :mod:`solve_network`.
 """
+import importlib
 import logging
+import os
 import re
 
 import numpy as np
@@ -792,6 +794,11 @@ def extra_functionality(n, snapshots):
         add_carbon_budget_constraint(n, snapshots)
         add_retrofit_gas_boiler_constraint(n, snapshots)
 
+    if snakemake.params.custom_extra_functionality:
+        source_path = snakemake.params.custom_extra_functionality
+        module_name = os.path.splitext(os.path.basename(source_path))[0]
+        module = importlib.import_module(module_name)
+        module.custom_extra_functionality(n, snapshots)
 
 def solve_network(n, config, solving, opts="", **kwargs):
     set_of_options = solving["solver"]["options"]

From fba320bfa7ae05a86e567426848577d42a25b337 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 29 Dec 2023 16:20:24 +0000
Subject: [PATCH 02/10] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/solve_network.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 539c4e72e..d79a63429 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -800,6 +800,7 @@ def extra_functionality(n, snapshots):
         module = importlib.import_module(module_name)
         module.custom_extra_functionality(n, snapshots)
 
+
 def solve_network(n, config, solving, opts="", **kwargs):
     set_of_options = solving["solver"]["options"]
     cf_solving = solving["options"]

From 4b6dd2908324eda4c1722cb9ba41d330df6da443 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Fri, 29 Dec 2023 17:23:11 +0100
Subject: [PATCH 03/10] add dummy file and assert path exists

---
 data/custom_extra_functionality.py | 9 +++++++++
 scripts/solve_network.py           | 1 +
 2 files changed, 10 insertions(+)
 create mode 100644 data/custom_extra_functionality.py

diff --git a/data/custom_extra_functionality.py b/data/custom_extra_functionality.py
new file mode 100644
index 000000000..98b0c0266
--- /dev/null
+++ b/data/custom_extra_functionality.py
@@ -0,0 +1,9 @@
+# SPDX-FileCopyrightText: : 2023- The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+def custom_extra_functionality(n, snapshots):
+    """
+    Add custom extra functionality constraints.
+    """
+    pass
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 539c4e72e..ed28c51c6 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -796,6 +796,7 @@ def extra_functionality(n, snapshots):
 
     if snakemake.params.custom_extra_functionality:
         source_path = snakemake.params.custom_extra_functionality
+        assert os.path.exists(source_path), f"{source_path} does not exist"
         module_name = os.path.splitext(os.path.basename(source_path))[0]
         module = importlib.import_module(module_name)
         module.custom_extra_functionality(n, snapshots)

From 876a28b688719273cc8fd9531845ae78c0e7edbe Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 29 Dec 2023 16:23:52 +0000
Subject: [PATCH 04/10] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 data/custom_extra_functionality.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/data/custom_extra_functionality.py b/data/custom_extra_functionality.py
index 98b0c0266..0ac24cea9 100644
--- a/data/custom_extra_functionality.py
+++ b/data/custom_extra_functionality.py
@@ -1,7 +1,9 @@
+# -*- coding: utf-8 -*-
 # SPDX-FileCopyrightText: : 2023- The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 
+
 def custom_extra_functionality(n, snapshots):
     """
     Add custom extra functionality constraints.

From a5ba2565a0abfcbe84d25c75a66fd2639bf08ca1 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Fri, 29 Dec 2023 17:30:03 +0100
Subject: [PATCH 05/10] correct config location

---
 rules/solve_myopic.smk    | 2 +-
 rules/solve_overnight.smk | 2 +-
 rules/solve_perfect.smk   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk
index 217547b97..afa8ad2c5 100644
--- a/rules/solve_myopic.smk
+++ b/rules/solve_myopic.smk
@@ -89,7 +89,7 @@ rule solve_sector_network_myopic:
             "co2_sequestration_potential", 200
         ),
         custom_extra_functionality=workflow.source_path(
-            config["solver"]["options"].get("custom_extra_functionality", "")
+            config["solving"]["options"].get("custom_extra_functionality", "")
         ),
     input:
         network=RESULTS
diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk
index 8ac56db8c..fc2f74dfa 100644
--- a/rules/solve_overnight.smk
+++ b/rules/solve_overnight.smk
@@ -12,7 +12,7 @@ rule solve_sector_network:
             "co2_sequestration_potential", 200
         ),
         custom_extra_functionality=workflow.source_path(
-            config["solver"]["options"].get("custom_extra_functionality", "")
+            config["solving"]["options"].get("custom_extra_functionality", "")
         ),
     input:
         network=RESULTS
diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk
index 322ced8dd..63be5cc19 100644
--- a/rules/solve_perfect.smk
+++ b/rules/solve_perfect.smk
@@ -119,7 +119,7 @@ rule solve_sector_network_perfect:
             "co2_sequestration_potential", 200
         ),
         custom_extra_functionality=workflow.source_path(
-            config["solver"]["options"].get("custom_extra_functionality", "")
+            config["solving"]["options"].get("custom_extra_functionality", "")
         ),
     input:
         network=RESULTS

From 79ca64382b56e69315e641c9441e7a036a97313a Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 31 Dec 2023 14:15:37 +0100
Subject: [PATCH 06/10] correct path for custom_extra_functionality

---
 config/config.default.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index 0ff742e70..b8945f754 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -627,7 +627,7 @@ solving:
     skip_iterations: true
     rolling_horizon: false
     seed: 123
-    custom_extra_functionality: "data/custom_extra_functionality.py"
+    custom_extra_functionality: "../data/custom_extra_functionality.py"
     # options that go into the optimize function
     track_iterations: false
     min_iterations: 4

From f28e9b47d26cc108c99103c3ce63b34d0ed2011d Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Tue, 2 Jan 2024 16:00:41 +0100
Subject: [PATCH 07/10] add custom_extra_functionality param to
 solve_electricity rule

---
 rules/solve_electricity.smk | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk
index c396ebd5a..2c9560971 100644
--- a/rules/solve_electricity.smk
+++ b/rules/solve_electricity.smk
@@ -11,6 +11,9 @@ rule solve_network:
         co2_sequestration_potential=config["sector"].get(
             "co2_sequestration_potential", 200
         ),
+        custom_extra_functionality=workflow.source_path(
+            config["solving"]["options"].get("custom_extra_functionality", "")
+        ),
     input:
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
         config=RESULTS + "config.yaml",

From 340bf778498a4a6ceffe246b8ab3245a9ccc84f7 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Tue, 2 Jan 2024 16:01:28 +0100
Subject: [PATCH 08/10] clarify that source_path is relative to  directory

---
 doc/configtables/solving.csv | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/configtables/solving.csv b/doc/configtables/solving.csv
index 9d47c0430..dcff54e42 100644
--- a/doc/configtables/solving.csv
+++ b/doc/configtables/solving.csv
@@ -6,7 +6,7 @@ options,,,
 -- skip_iterations,bool,"{'true','false'}","Skip iterating, do not update impedances of branches. Defaults to true."
 -- rolling_horizon,bool,"{'true','false'}","Whether to optimize the network in a rolling horizon manner, where the snapshot range is split into slices of size `horizon` which are solved consecutively."
 -- seed,--,int,Random seed for increased deterministic behaviour.
--- custom_extra_functionality,--,str,Path to a Python file with custom extra functionality code to be injected into the solving rules of the workflow.
+-- custom_extra_functionality,--,str,Path to a Python file with custom extra functionality code to be injected into the solving rules of the workflow relative to ``rules`` directory.
 -- track_iterations,bool,"{'true','false'}",Flag whether to store the intermediate branch capacities and objective function values are recorded for each iteration in ``network.lines['s_nom_opt_X']`` (where ``X`` labels the iteration)
 -- min_iterations,--,int,Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run.
 -- max_iterations,--,int,Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run.

From d145758fb7ff4bf126ddada8eec6d8f942c93f4f Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Thu, 4 Jan 2024 09:00:31 +0100
Subject: [PATCH 09/10] gracefully handle absent extra_functionality file; add
 file to path

---
 rules/common.smk            | 7 +++++++
 rules/solve_electricity.smk | 4 +---
 rules/solve_myopic.smk      | 4 +---
 rules/solve_overnight.smk   | 5 +----
 rules/solve_perfect.smk     | 4 +---
 scripts/solve_network.py    | 2 ++
 6 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/rules/common.smk b/rules/common.smk
index 2c8cf69c1..44e3a8078 100644
--- a/rules/common.smk
+++ b/rules/common.smk
@@ -28,6 +28,13 @@ def memory(w):
         return int(factor * (10000 + 195 * int(w.clusters)))
 
 
+def input_custom_extra_functionality(w):
+    path = config["solving"]["options"].get("custom_extra_functionality", False)
+    if path:
+        return workflow.source_path(path)
+    return []
+
+
 # Check if the workflow has access to the internet by trying to access the HEAD of specified url
 def has_internet_access(url="www.zenodo.org") -> bool:
     import http.client as http_client
diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk
index 2c9560971..7f6092be2 100644
--- a/rules/solve_electricity.smk
+++ b/rules/solve_electricity.smk
@@ -11,9 +11,7 @@ rule solve_network:
         co2_sequestration_potential=config["sector"].get(
             "co2_sequestration_potential", 200
         ),
-        custom_extra_functionality=workflow.source_path(
-            config["solving"]["options"].get("custom_extra_functionality", "")
-        ),
+        custom_extra_functionality=input_custom_extra_functionality,
     input:
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
         config=RESULTS + "config.yaml",
diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk
index afa8ad2c5..7ca8857db 100644
--- a/rules/solve_myopic.smk
+++ b/rules/solve_myopic.smk
@@ -88,9 +88,7 @@ rule solve_sector_network_myopic:
         co2_sequestration_potential=config["sector"].get(
             "co2_sequestration_potential", 200
         ),
-        custom_extra_functionality=workflow.source_path(
-            config["solving"]["options"].get("custom_extra_functionality", "")
-        ),
+        custom_extra_functionality=input_custom_extra_functionality,
     input:
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk
index fc2f74dfa..8686b205f 100644
--- a/rules/solve_overnight.smk
+++ b/rules/solve_overnight.smk
@@ -2,7 +2,6 @@
 #
 # SPDX-License-Identifier: MIT
 
-
 rule solve_sector_network:
     params:
         solving=config["solving"],
@@ -11,9 +10,7 @@ rule solve_sector_network:
         co2_sequestration_potential=config["sector"].get(
             "co2_sequestration_potential", 200
         ),
-        custom_extra_functionality=workflow.source_path(
-            config["solving"]["options"].get("custom_extra_functionality", "")
-        ),
+        custom_extra_functionality=input_custom_extra_functionality,
     input:
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk
index 63be5cc19..a7856fa93 100644
--- a/rules/solve_perfect.smk
+++ b/rules/solve_perfect.smk
@@ -118,9 +118,7 @@ rule solve_sector_network_perfect:
         co2_sequestration_potential=config["sector"].get(
             "co2_sequestration_potential", 200
         ),
-        custom_extra_functionality=workflow.source_path(
-            config["solving"]["options"].get("custom_extra_functionality", "")
-        ),
+        custom_extra_functionality=input_custom_extra_functionality,
     input:
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 5a0455771..2f170dff3 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -30,6 +30,7 @@
 import logging
 import os
 import re
+import sys
 
 import numpy as np
 import pandas as pd
@@ -831,6 +832,7 @@ def extra_functionality(n, snapshots):
     if snakemake.params.custom_extra_functionality:
         source_path = snakemake.params.custom_extra_functionality
         assert os.path.exists(source_path), f"{source_path} does not exist"
+        sys.path.append(os.path.dirname(source_path))
         module_name = os.path.splitext(os.path.basename(source_path))[0]
         module = importlib.import_module(module_name)
         module.custom_extra_functionality(n, snapshots)

From 60493fc55829ddc95bd8d55d35b0f505cef5f624 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 4 Jan 2024 08:01:00 +0000
Subject: [PATCH 10/10] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 rules/solve_overnight.smk | 1 +
 1 file changed, 1 insertion(+)

diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk
index 8686b205f..a3fed042a 100644
--- a/rules/solve_overnight.smk
+++ b/rules/solve_overnight.smk
@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MIT
 
+
 rule solve_sector_network:
     params:
         solving=config["solving"],