Manage extensions' XML import/export with subnetworks

Signed-off-by: Olivier Perrin <olivier.perrin@rte-france.com>

commons-test/src/main/java/com/powsybl/commons/test/AbstractConverterTest.java

@@ -60,10 +60,12 @@ protected <T> Path writeTest(T data, BiConsumer<T, Path> out, BiConsumer<InputSt
     }
 
     protected <T> T roundTripTest(T data, BiConsumer<T, Path> out, Function<Path, T> in, BiConsumer<InputStream, InputStream> compare, String ref) throws IOException {
+        // Export the data and check the result with the reference
         Path xmlFile = writeTest(data, out, compare, ref);
         try (InputStream is1 = Files.newInputStream(xmlFile)) {
             compare.accept(getClass().getResourceAsStream(ref), is1);
         }
+        // Read the exported data, export the retrieved data and check the result with the reference
         T data2 = in.apply(xmlFile);
         Path xmlFile2 = tmpDir.resolve("data2");
         out.accept(data2, xmlFile2);

commons/src/main/java/com/powsybl/commons/extensions/ExtensionProviders.java

@@ -9,10 +9,7 @@
 import com.powsybl.commons.PowsyblException;
 import com.powsybl.commons.util.ServiceLoaderCache;
 
-import java.util.Collection;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
+import java.util.*;
 import java.util.stream.Collectors;
 
 /**
@@ -48,7 +45,8 @@ private ExtensionProviders(Class<T> clazz, String categoryName, Set<String> exte
         Objects.requireNonNull(clazz);
         Objects.requireNonNull(categoryName);
 
-        providers = new ServiceLoaderCache<>(clazz).getServices().stream()
+        List<T> services = new ServiceLoaderCache<>(clazz).getServices();
+        providers = services.stream()
                 .filter(s -> s.getCategoryName().equals(categoryName) && (extensionNames == null || extensionNames.contains(s.getExtensionName())))
                 .collect(Collectors.toMap(T::getExtensionName, e -> e));
     }

iidm/iidm-impl/src/main/java/com/powsybl/iidm/network/impl/AbstractNetwork.java

@@ -77,6 +77,7 @@ protected String getTypeDescription() {
      * @param to the destination network
      */
     protected static void transferExtensions(Network from, Network to) {
+        // Only well-defined extensions (with an interface) are transferred
         new ArrayList<>(from.getExtensions())
                 .forEach(e -> Arrays.stream(e.getClass().getInterfaces())
                         .filter(c -> Objects.nonNull(from.getExtension(c)))

iidm/iidm-xml-converter/src/main/java/com/powsybl/iidm/xml/NetworkXml.java

@@ -213,8 +213,15 @@ private static String getNamespaceUri(ExtensionXmlSerializer extensionXmlSeriali
     }
 
     private static void writeExtensions(Network n, NetworkXmlWriterContext context) throws XMLStreamException {
-        for (Identifiable<?> identifiable : IidmXmlUtil.sorted(n.getIdentifiables(), context.getOptions())) {
-            if (!context.isExportedEquipment(identifiable)) {
+        Collection<Identifiable<?>> identifiables = n.getIdentifiables();
+        if (supportSubnetworksExport(context) && n.getParentNetwork() != n) {
+            // Subnetworks are not in their own identifiables.
+            identifiables = new HashSet<>(identifiables);
+            identifiables.add(n);
+        }
+
+        for (Identifiable<?> identifiable : IidmXmlUtil.sorted(identifiables, context.getOptions())) {
+            if (!context.isExportedEquipment(identifiable) || !retainElement(identifiable, n, context)) {
                 continue;
             }
             Collection<? extends Extension<? extends Identifiable<?>>> extensions = identifiable.getExtensions().stream()
@@ -266,9 +273,11 @@ private static void writeRootElement(Network n, NetworkXmlWriterContext context)
         boolean isSubnetwork = n.getParentNetwork() != n;
         context.getWriter().setPrefix(IIDM_PREFIX, namespaceUri);
         context.getWriter().writeStartElement(namespaceUri, isSubnetwork ? SUBNETWORK_ROOT_ELEMENT_NAME : NETWORK_ROOT_ELEMENT_NAME);
-        context.getWriter().writeNamespace(IIDM_PREFIX, namespaceUri);
-        if (!options.withNoExtension()) {
-            writeExtensionNamespaces(n, context);
+        if (!isSubnetwork) {
+            context.getWriter().writeNamespace(IIDM_PREFIX, namespaceUri);
+            if (!options.withNoExtension()) {
+                writeExtensionNamespaces(n, context);
+            }
         }
         writeMainAttributes(n, context);
     }
@@ -299,7 +308,7 @@ private static void writeSubNetworks(Network n, NetworkXmlWriterContext context)
 
     private static void writeVoltageLevels(Network n, NetworkXmlWriterContext context) throws XMLStreamException {
         for (VoltageLevel voltageLevel : IidmXmlUtil.sorted(n.getVoltageLevels(), context.getOptions())) {
-            if (retain(voltageLevel, n, context) && voltageLevel.getSubstation().isEmpty()) {
+            if (retainElement(voltageLevel, n, context) && voltageLevel.getSubstation().isEmpty()) {
                 IidmXmlUtil.assertMinimumVersion(NETWORK_ROOT_ELEMENT_NAME, VoltageLevelXml.ROOT_ELEMENT_NAME,
                         IidmXmlUtil.ErrorMessage.NOT_SUPPORTED, IidmXmlVersion.V_1_6, context);
                 VoltageLevelXml.INSTANCE.write(voltageLevel, n, context);
@@ -309,7 +318,7 @@ private static void writeVoltageLevels(Network n, NetworkXmlWriterContext contex
 
     private static void writeSubstations(Network n, NetworkXmlWriterContext context) throws XMLStreamException {
         for (Substation s : IidmXmlUtil.sorted(n.getSubstations(), context.getOptions())) {
-            if (retain(s, n, context)) {
+            if (retainElement(s, n, context)) {
                 SubstationXml.INSTANCE.write(s, n, context);
             }
         }
@@ -318,14 +327,14 @@ private static void writeSubstations(Network n, NetworkXmlWriterContext context)
     private static void writeTransformers(Network n, NetworkXmlWriterContext context) throws XMLStreamException {
         BusFilter filter = context.getFilter();
         for (TwoWindingsTransformer twt : IidmXmlUtil.sorted(n.getTwoWindingsTransformers(), context.getOptions())) {
-            if (twt.getSubstation().isEmpty() && retain(twt, n, context) && filter.test(twt)) {
+            if (twt.getSubstation().isEmpty() && retainElement(twt, n, context) && filter.test(twt)) {
                 IidmXmlUtil.assertMinimumVersion(NETWORK_ROOT_ELEMENT_NAME, TwoWindingsTransformerXml.ROOT_ELEMENT_NAME,
                         IidmXmlUtil.ErrorMessage.NOT_SUPPORTED, IidmXmlVersion.V_1_6, context);
                 TwoWindingsTransformerXml.INSTANCE.write(twt, n, context);
             }
         }
         for (ThreeWindingsTransformer twt : IidmXmlUtil.sorted(n.getThreeWindingsTransformers(), context.getOptions())) {
-            if (twt.getSubstation().isEmpty() && retain(twt, n, context) && filter.test(twt)) {
+            if (twt.getSubstation().isEmpty() && retainElement(twt, n, context) && filter.test(twt)) {
                 IidmXmlUtil.assertMinimumVersion(NETWORK_ROOT_ELEMENT_NAME, ThreeWindingsTransformerXml.ROOT_ELEMENT_NAME,
                         IidmXmlUtil.ErrorMessage.NOT_SUPPORTED, IidmXmlVersion.V_1_6, context);
                 ThreeWindingsTransformerXml.INSTANCE.write(twt, n, context);
@@ -336,7 +345,7 @@ private static void writeTransformers(Network n, NetworkXmlWriterContext context
     private static void writeLines(Network n, NetworkXmlWriterContext context) throws XMLStreamException {
         BusFilter filter = context.getFilter();
         for (Line l : IidmXmlUtil.sorted(n.getLines(), context.getOptions())) {
-            if (!retain(l, n, context) || !filter.test(l)) {
+            if (!retainElement(l, n, context) || !filter.test(l)) {
                 continue;
             }
             LineXml.INSTANCE.write(l, n, context);
@@ -346,7 +355,7 @@ private static void writeLines(Network n, NetworkXmlWriterContext context) throw
     private static void writeTieLines(Network n, NetworkXmlWriterContext context) throws XMLStreamException {
         BusFilter filter = context.getFilter();
         for (TieLine l : IidmXmlUtil.sorted(n.getTieLines(), context.getOptions())) {
-            if (!retain(l, n, context) || !filter.test(l)) {
+            if (!retainElement(l, n, context) || !filter.test(l)) {
                 continue;
             }
             TieLineXml.INSTANCE.write(l, n, context);
@@ -356,14 +365,17 @@ private static void writeTieLines(Network n, NetworkXmlWriterContext context) th
     private static void writeHvdcLines(Network n, NetworkXmlWriterContext context) throws XMLStreamException {
         BusFilter filter = context.getFilter();
         for (HvdcLine l : IidmXmlUtil.sorted(n.getHvdcLines(), context.getOptions())) {
-            if (!retain(l, n, context) || !filter.test(l.getConverterStation1()) || !filter.test(l.getConverterStation2())) {
+            if (!retainElement(l, n, context) || !filter.test(l.getConverterStation1()) || !filter.test(l.getConverterStation2())) {
                 continue;
             }
             HvdcLineXml.INSTANCE.write(l, n, context);
         }
     }
 
     private static void write(Network network, NetworkXmlWriterContext context) throws XMLStreamException {
+        // consider the network has been exported so its extensions will be written
+        // (should be done before extensions are written)
+        context.addExportedEquipment(network);
         writeRootElement(network, context);
         writeBaseNetwork(network, context);
         writeExtensions(network, context);
@@ -383,16 +395,21 @@ public static Anonymizer write(Network n, ExportOptions options, OutputStream os
         }
     }
 
-    private static boolean retain(Identifiable<?> equipment, Network n, NetworkXmlWriterContext context) {
-        return context.getVersion().compareTo(IidmXmlVersion.V_1_11) < 0 || equipment.getParentNetwork() == n;
+    private static boolean retainElement(Identifiable<?> element, Network n, NetworkXmlWriterContext context) {
+        return !supportSubnetworksExport(context)
+                || n.getId().equals(element.getId()) // the element is the network
+                || element.getParentNetwork() == n; // the element is directly in the network (not in one of its subnetworks)
+    }
+
+    private static boolean supportSubnetworksExport(NetworkXmlWriterContext context) {
+        return context.getVersion().compareTo(IidmXmlVersion.V_1_11) >= 0;
     }
 
     private static NetworkXmlWriterContext createContext(Network n, ExportOptions options, XMLStreamWriter writer) {
         BusFilter filter = BusFilter.create(n, options);
         Anonymizer anonymizer = options.isAnonymized() ? new SimpleAnonymizer() : null;
         IidmXmlVersion version = options.getVersion() == null ? IidmXmlConstants.CURRENT_IIDM_XML_VERSION : IidmXmlVersion.of(options.getVersion(), ".");
         NetworkXmlWriterContext context = new NetworkXmlWriterContext(anonymizer, writer, options, filter, version, n.getValidationLevel() == ValidationLevel.STEADY_STATE_HYPOTHESIS);
-        context.addExportedEquipment(n); // consider the network has been exported so its extensions will be written also
         return context;
     }
 
@@ -541,9 +558,17 @@ private static void readElements(NetworkFactory networkFactory, XMLStreamReader
 
             case EXTENSION_ELEMENT_NAME:
                 String id2 = context.getAnonymizer().deanonymizeString(reader.getAttributeValue(null, "id"));
-                Identifiable identifiable = networks.peek().getIdentifiable(id2);
+                Network network = networks.peek();
+                Identifiable identifiable = network.getIdentifiable(id2);
                 if (identifiable == null) {
-                    throw new PowsyblException("Identifiable " + id2 + " not found");
+                    if (network.getId().equals(id2)) {
+                        // Subnetworks are not in the index of the networks (root network or subnetworks).
+                        // To support network extensions on subnetworks, we check if the unknown identifiable
+                        // is the current network.
+                        identifiable = network;
+                    } else {
+                        throw new PowsyblException("Identifiable " + id2 + " not found");
+                    }
                 }
                 readExtensions(identifiable, context, extensionNamesNotFound);
                 break;

iidm/iidm-xml-converter/src/test/java/com/powsybl/iidm/xml/IdentifiableExtensionXmlSerializerTest.java

@@ -6,23 +6,19 @@
  */
 package com.powsybl.iidm.xml;
 
-import com.google.auto.service.AutoService;
 import com.powsybl.commons.PowsyblException;
 import com.powsybl.commons.datasource.MemDataSource;
 import com.powsybl.commons.extensions.AbstractExtension;
-import com.powsybl.commons.extensions.AbstractExtensionXmlSerializer;
-import com.powsybl.commons.extensions.ExtensionXmlSerializer;
-import com.powsybl.commons.xml.XmlReaderContext;
-import com.powsybl.commons.xml.XmlWriterContext;
 import com.powsybl.iidm.network.Load;
 import com.powsybl.iidm.network.Network;
 import com.powsybl.iidm.network.test.EurostagTutorialExample1Factory;
 import com.powsybl.iidm.network.test.LoadZipModel;
 import com.powsybl.iidm.network.test.MultipleExtensionsTestNetworkFactory;
 import com.powsybl.iidm.network.test.TerminalMockExt;
+import com.powsybl.iidm.xml.extensions.util.NetworkSourceExtension;
+import com.powsybl.iidm.xml.extensions.util.NetworkSourceExtensionImpl;
 import org.junit.jupiter.api.Test;
 
-import javax.xml.stream.XMLStreamException;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -115,50 +111,11 @@ void testExtensionWithoutSerializerDoNotThrowException() throws IOException {
         }
     }
 
-    // Define a network extension with XML serializer
-    static class NetworkSourceExtension extends AbstractExtension<Network> {
-
-        NetworkSourceExtension(String sourceData) {
-            this.sourceData = sourceData;
-        }
-
-        @Override
-        public String getName() {
-            return "networkSource";
-        }
-
-        String getSourceData() {
-            return sourceData;
-        }
-
-        private final String sourceData;
-    }
-
-    @AutoService(ExtensionXmlSerializer.class)
-    public static class NetworkSourceExtensionXmlSerializer extends AbstractExtensionXmlSerializer<Network, NetworkSourceExtension> {
-
-        public NetworkSourceExtensionXmlSerializer() {
-            super("networkSource", "network", NetworkSourceExtension.class, false, "networkSource.xsd",
-                    "http://www.itesla_project.eu/schema/iidm/ext/networksource/1_0", "extNetworkSource");
-        }
-
-        @Override
-        public void write(NetworkSourceExtension networkSource, XmlWriterContext context) throws XMLStreamException {
-            context.getWriter().writeAttribute("sourceData", networkSource.getSourceData());
-        }
-
-        @Override
-        public NetworkSourceExtension read(Network network, XmlReaderContext context) {
-            String sourceData = context.getReader().getAttributeValue(null, "sourceData");
-            return new NetworkSourceExtension(sourceData);
-        }
-    }
-
     @Test
     void testNetworkSourceExtension() throws IOException {
         Network network = EurostagTutorialExample1Factory.create();
         String sourceData = "eurostag-tutorial-example1-created-from-IIDM-API";
-        NetworkSourceExtension source = new NetworkSourceExtension(sourceData);
+        NetworkSourceExtension source = new NetworkSourceExtensionImpl(sourceData);
         network.addExtension(NetworkSourceExtension.class, source);
         byte[] buffer;
         try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {

iidm/iidm-xml-converter/src/test/java/com/powsybl/iidm/xml/NetworkXmlTest.java

@@ -12,10 +12,9 @@
 import com.powsybl.commons.xml.XmlReaderContext;
 import com.powsybl.commons.xml.XmlWriterContext;
 import com.powsybl.iidm.network.*;
-import com.powsybl.iidm.network.test.EurostagTutorialExample1Factory;
-import com.powsybl.iidm.network.test.BusbarSectionExt;
-import com.powsybl.iidm.network.test.ScadaNetworkFactory;
-import com.powsybl.iidm.network.test.NetworkTest1Factory;
+import com.powsybl.iidm.network.test.*;
+import com.powsybl.iidm.xml.extensions.util.NetworkSourceExtension;
+import com.powsybl.iidm.xml.extensions.util.NetworkSourceExtensionImpl;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.Test;
 
@@ -224,23 +223,23 @@ void checkWithSpecificEncoding() throws IOException {
 
     @Test
     void roundTripWithSubNetworksTest() throws IOException {
-        Network merged = Network.create("Merged", "hybrid");
         Network n1 = createNetwork(1);
         Network n2 = createNetwork(2);
-        merged.setCaseDate(DateTime.parse("2013-01-15T18:40:00+01:00"));
         n1.setCaseDate(DateTime.parse("2013-01-15T18:41:00+01:00"));
         n2.setCaseDate(DateTime.parse("2013-01-15T18:42:00+01:00"));
-        merged.merge(n1, n2);
 
-        //TODO
-        // - Remove the "xmlns:iidm" attribute for subnetworks
+        Network merged = Network.create("Merged", n1, n2);
+        merged.setCaseDate(DateTime.parse("2013-01-15T18:40:00+01:00"));
+        // add an extension at root network level
+        NetworkSourceExtension source = new NetworkSourceExtensionImpl("Source_0");
+        merged.addExtension(NetworkSourceExtension.class, source);
 
         for (IidmXmlVersion version : IidmXmlVersion.values()) {
             if (version.compareTo(IidmXmlVersion.V_1_5) >= 0) {
                 roundTripXmlTest(merged,
                     (n, p) -> NetworkXml.writeAndValidate(n,
                         new ExportOptions().setSorted(true).setVersion(version.toString(".")), p),
-                    NetworkXml::read,
+                    NetworkXml::validateAndRead,
                     getVersionedNetworkPath("subnetworks.xml", version));
             }
         }
@@ -277,6 +276,41 @@ private Network createNetwork(int num) {
                 .setB(5.0)
                 .setUcteXnodeCode("code")
                 .add();
+
+        // Add an extension on the network and on an inner element
+        NetworkSourceExtension source = new NetworkSourceExtensionImpl("Source_" + num);
+        network.addExtension(NetworkSourceExtension.class, source);
+
+        if (num == 1) {
+            Generator generator = vl1.newGenerator()
+                    .setId("GEN")
+                    .setBus(busId)
+                    .setConnectableBus(busId)
+                    .setMinP(-9999.99)
+                    .setMaxP(9999.99)
+                    .setVoltageRegulatorOn(true)
+                    .setTargetV(24.5)
+                    .setTargetP(607.0)
+                    .setTargetQ(301.0)
+                    .add();
+            generator.newMinMaxReactiveLimits()
+                    .setMinQ(-9999.99)
+                    .setMaxQ(9999.99)
+                    .add();
+        } else if (num == 2) {
+            vl1.newLoad()
+                    .setId("LOAD")
+                    .setBus(busId)
+                    .setConnectableBus(busId)
+                    .setP0(600.0)
+                    .setQ0(200.0)
+                    .add();
+
+            // Add an extension on an inner element
+            Load load = network.getLoad("LOAD");
+            TerminalMockExt terminalMockExt = new TerminalMockExt(load);
+            load.addExtension(TerminalMockExt.class, terminalMockExt);
+        }
         return network;
     }
 }

iidm/iidm-xml-converter/src/test/java/com/powsybl/iidm/xml/extensions/util/NetworkSourceExtension.java

@@ -0,0 +1,25 @@
+/**
+ * Copyright (c) 2023, RTE (http://www.rte-france.com)
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+package com.powsybl.iidm.xml.extensions.util;
+
+import com.powsybl.commons.extensions.Extension;
+import com.powsybl.iidm.network.Network;
+
+/**
+ * @author Olivier Perrin <olivier.perrin at rte-france.com>
+ */
+public interface NetworkSourceExtension extends Extension<Network> {
+    String NAME = "networkSource";
+
+    String getSourceData();
+
+    @Override
+    default String getName() {
+        return NAME;
+    }
+}