CGMES : Implement a pre-processor that completes the CGMES model with missing containers before starting the conversion

cgmes/cgmes-completion/pom.xml

@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Copyright (c) 2023, RTE (http://www.rte-france.com)
+    This Source Code Form is subject to the terms of the Mozilla Public
+    License, v. 2.0. If a copy of the MPL was not distributed with this
+    file, You can obtain one at http://mozilla.org/MPL/2.0/.
+    SPDX-License-Identifier: MPL-2.0
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.powsybl</groupId>
+        <artifactId>powsybl-cgmes</artifactId>
+        <version>5.4.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>powsybl-cgmes-completion</artifactId>
+    <name>CGMES network model completion</name>
+    <description>CGMES (Common Grid Model Exchange Specification). Complete missing data in input instance files
+    </description>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <archive>
+                        <manifestEntries>
+                            <Automatic-Module-Name>com.powsybl.cgmes.completion</Automatic-Module-Name>
+                        </manifestEntries>
+                    </archive>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <!-- Compilation dependencies -->
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>powsybl-cgmes-conversion</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <!-- Test dependencies -->
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.jimfs</groupId>
+            <artifactId>jimfs</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>log4j-over-slf4j</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>powsybl-config-test</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>powsybl-cgmes-conformity</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>powsybl-iidm-impl</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- Test dependencies include all considered triple store engines -->
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>powsybl-triple-store-impl-rdf4j</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+</project>

cgmes/cgmes-completion/src/main/java/com/powsybl/cgmes/completion/CreateMissingContainersPreProcessor.java

@@ -0,0 +1,245 @@
+/**
+ * Copyright (c) 2023, RTE (http://www.rte-france.com)
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+package com.powsybl.cgmes.completion;
+
+import com.google.auto.service.AutoService;
+import com.powsybl.cgmes.conversion.CgmesImportPreProcessor;
+import com.powsybl.cgmes.conversion.export.CgmesExportContext;
+import com.powsybl.cgmes.conversion.export.CgmesExportUtil;
+import com.powsybl.cgmes.conversion.export.elements.*;
+import com.powsybl.cgmes.extensions.CgmesTopologyKind;
+import com.powsybl.cgmes.extensions.CimCharacteristicsAdder;
+import com.powsybl.cgmes.model.CgmesModel;
+import com.powsybl.cgmes.model.CgmesNames;
+import com.powsybl.cgmes.model.triplestore.CgmesModelTripleStore;
+import com.powsybl.commons.PowsyblException;
+import com.powsybl.commons.config.PlatformConfig;
+import com.powsybl.commons.datasource.ZipFileDataSource;
+import com.powsybl.commons.parameters.Parameter;
+import com.powsybl.commons.parameters.ParameterDefaultValueConfig;
+import com.powsybl.commons.parameters.ParameterType;
+import com.powsybl.commons.reporter.Reporter;
+import com.powsybl.commons.xml.XmlUtil;
+import com.powsybl.iidm.network.Network;
+import com.powsybl.iidm.network.NetworkFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * @author Luma Zamarreño <zamarrenolm at aia.es>
+ */
+@AutoService(CgmesImportPreProcessor.class)
+public class CreateMissingContainersPreProcessor implements CgmesImportPreProcessor {
+
+    public static final String NAME = "createMissingContainers";
+    public static final String FIXES_FOLDER_NAME = "iidm.import.cgmes.fixes-for-missing-containers-folder";
+    public static final double DEFAULT_NOMINAL_VALUE_FOR_MISSING_VOLTAGE_LEVELS = 100.0;
+
+    private static final Logger LOG = LoggerFactory.getLogger(CreateMissingContainersPreProcessor.class);
+    private static final Parameter FIXES_FOLDER_NAME_PARAMETER = new Parameter(FIXES_FOLDER_NAME,
+            ParameterType.STRING,
+            "Folder where zip files containing fixes will be created: one zip for each imported network missing data",
+            null);
+
+    private final PlatformConfig platformConfig;
+    private final ParameterDefaultValueConfig defaultValueConfig;
+
+    public CreateMissingContainersPreProcessor(PlatformConfig platformConfig) {
+        Objects.requireNonNull(platformConfig);
+        this.platformConfig = platformConfig;
+        defaultValueConfig = new ParameterDefaultValueConfig(platformConfig);
+    }
+
+    public CreateMissingContainersPreProcessor() {
+        this(PlatformConfig.defaultConfig());
+    }
+
+    private static String nameFor(CgmesModel cgmes) {
+        try {
+            return new URI(cgmes.getBasename()).getAuthority();
+        } catch (URISyntaxException e) {
+            return cgmes.modelId();
+        }
+    }
+
+    private static void prepareAndReadFixesUsingFolder(CgmesModel cgmes, String basename, Path fixesFolder) {
+        if (!Files.isDirectory(fixesFolder)) {
+            LOG.error("Output folder is not a directory {}. Skipping post processor.", fixesFolder);
+            return;
+        }
+        Path fixesFile = fixesFolder.resolve(basename + ".zip");
+        // Check the file will be writable
+        try {
+            Files.deleteIfExists(fixesFile);
+            Files.createFile(fixesFile);
+        } catch (IOException e) {
+            LOG.error("Output file {} is not writable. Skipping post processor.", fixesFile);
+            return;
+        }
+        if (LOG.isInfoEnabled()) {
+            LOG.info("Execute {} pre processor on CGMES model {}. Output to file {}", NAME, cgmes.modelId(), fixesFile);
+        }
+        prepareAndReadFixesUsingZipFile(cgmes, basename, fixesFile);
+    }
+
+    private static void prepareAndReadFixesUsingZipFile(CgmesModel cgmes, String basename, Path fixesFile) {
+        // Assume all containers missing are voltage levels and create proper objects for them (substation, regions, ...)
+        Set<String> missingVoltageLevels = findMissingVoltageLevels(cgmes);
+        LOG.info("Missing voltage levels: {}", missingVoltageLevels);
+        if (!missingVoltageLevels.isEmpty()) {
+            buildZipFileWithFixes(cgmes, missingVoltageLevels, fixesFile, basename);
+            cgmes.read(new ZipFileDataSource(fixesFile), Reporter.NO_OP);
+        }
+        Set<String> missingVoltageLevelsAfterFix = findMissingVoltageLevels(cgmes);
+        if (!missingVoltageLevelsAfterFix.isEmpty()) {
+            throw new IllegalStateException("Missing voltage levels after fix: " + missingVoltageLevelsAfterFix);
+        }
+        // The only containers without voltage level must be of type line
+        LOG.info("After the fixes have been applied, the only node containers without voltage level must be of type Line.");
+        LOG.info("Containers without voltage level that are not Lines will be reported as errors.");
+        cgmes.connectivityNodeContainers().stream()
+                .filter(c -> c.getId(CgmesNames.VOLTAGE_LEVEL) == null)
+                .filter(c -> !c.getLocal("connectivityNodeContainerType").equals("Line"))
+                .forEach(c -> LOG.error(c.getId(CgmesNames.CONNECTIVITY_NODE_CONTAINER)));
+    }
+
+    private static Set<String> findMissingVoltageLevels(CgmesModel cgmes) {
+        // check missing CN containers
+        Set<String> defined = cgmes.connectivityNodeContainers().stream().map(c -> c.getId(CgmesNames.CONNECTIVITY_NODE_CONTAINER)).collect(Collectors.toSet());
+        Set<String> referred = cgmes.connectivityNodes().stream().map(c -> c.getId(CgmesNames.CONNECTIVITY_NODE_CONTAINER)).collect(Collectors.toSet());
+        return referred.stream().filter(c -> !defined.contains(c)).collect(Collectors.toSet());
+    }
+
+    private static void buildZipFileWithFixes(CgmesModel cgmes, Set<String> missingVoltageLevels, Path fixesFile, String basename) {
+        Network network = prepareEmptyNetworkForExport(cgmes);
+        CgmesExportContext context = new CgmesExportContext(network);
+        try (ZipOutputStream zout = new ZipOutputStream(Files.newOutputStream(fixesFile))) {
+            zout.putNextEntry(new ZipEntry(basename + "_EQ.xml"));
+            XMLStreamWriter writer = XmlUtil.initializeWriter(true, "    ", zout);
+            writeHeader(writer, context);
+            RegionContainers regionContainers = writeRegionContainers(writer, context);
+            for (String missingVoltageLevel : missingVoltageLevels) {
+                writeMissingVoltageLevel(missingVoltageLevel, writer, context, regionContainers);
+            }
+            writer.writeEndDocument();
+            zout.closeEntry();
+        } catch (IOException | XMLStreamException x) {
+            throw new PowsyblException("Building file containing fixes for missing data", x);
+        }
+    }
+
+    private static Network prepareEmptyNetworkForExport(CgmesModel cgmes) {
+        Network network = NetworkFactory.findDefault().createNetwork("empty", "CGMES");
+        // We ensure that the fixes are exported to CGMES files with the same version of the input files
+        // To achieve it, we set the CIM characteristics of the empty Network created
+        if (cgmes instanceof CgmesModelTripleStore) {
+            network.newExtension(CimCharacteristicsAdder.class)
+                    .setTopologyKind(cgmes.isNodeBreaker() ? CgmesTopologyKind.NODE_BREAKER : CgmesTopologyKind.BUS_BRANCH)
+                    .setCimVersion(((CgmesModelTripleStore) cgmes).getCimVersion())
+                    .add();
+        }
+        return network;
+    }
+
+    private static RegionContainers writeRegionContainers(XMLStreamWriter writer, CgmesExportContext context) throws XMLStreamException {
+        String cimNamespace = context.getCim().getNamespace();
+
+        // An alternative to replicate this code would be to make public the method
+        // EquipmentExport::writeFictitiousSubstationFor and use it here.
+        // We could group all missing voltage levels in the same (fictitious) substation
+        RegionContainers regionContainers = new RegionContainers();
+        regionContainers.subGeographicalRegionId = CgmesExportUtil.getUniqueId();
+        String subGeographicalRegionName = "SGR fix for missing data";
+        regionContainers.geographicalRegionId = CgmesExportUtil.getUniqueId();
+        String geographicalRegionName = "GR fix for missing data";
+        SubGeographicalRegionEq.write(regionContainers.subGeographicalRegionId, subGeographicalRegionName, regionContainers.geographicalRegionId, cimNamespace, writer, context);
+        GeographicalRegionEq.write(regionContainers.geographicalRegionId, geographicalRegionName, cimNamespace, writer, context);
+        return regionContainers;
+    }
+
+    private static void writeMissingVoltageLevel(String voltageLevelId, XMLStreamWriter writer, CgmesExportContext context, RegionContainers regionContainers) throws XMLStreamException {
+        String cimNamespace = context.getCim().getNamespace();
+
+        // In a first approach,
+        // we do not have additional information about the voltage level,
+        // we create a different substation and base voltage for every missing voltage level
+        String voltageLevelName = voltageLevelId + " VL";
+        String substationId = CgmesExportUtil.getUniqueId();
+        String substationName = voltageLevelId + "SUB for missing VL " + voltageLevelId;
+        String baseVoltageId = CgmesExportUtil.getUniqueId();
+
+        VoltageLevelEq.write(voltageLevelId, voltageLevelName, Double.NaN, Double.NaN, substationId, baseVoltageId, cimNamespace, writer, context);
+        SubstationEq.write(substationId, substationName, regionContainers.subGeographicalRegionId, cimNamespace, writer, context);
+        BaseVoltageEq.write(baseVoltageId, DEFAULT_NOMINAL_VALUE_FOR_MISSING_VOLTAGE_LEVELS, cimNamespace, writer, context);
+    }
+
+    private static void writeHeader(XMLStreamWriter writer, CgmesExportContext context) throws XMLStreamException {
+        String cimNamespace = context.getCim().getNamespace();
+        String euNamespace = context.getCim().getEuNamespace();
+        CgmesExportUtil.writeRdfRoot(cimNamespace, context.getCim().getEuPrefix(), euNamespace, writer);
+        if (context.getCimVersion() >= 16) {
+            ModelDescriptionEq.write(writer, context.getEqModelDescription(), context);
+        }
+    }
+
+    private Path getFixesFolder() {
+        String fixesFolderName = Parameter.readString("CGMES", null, FIXES_FOLDER_NAME_PARAMETER, defaultValueConfig);
+        if (fixesFolderName == null) {
+            LOG.error("Executing {} pre processor. Missing the folder name for the output of files containing required fixes. Use the parameter {}.", NAME, FIXES_FOLDER_NAME_PARAMETER.getName());
+            return null;
+        }
+        Path fixesFolder = Paths.get(fixesFolderName);
+        if (fixesFolder.isAbsolute()) {
+            return fixesFolder;
+        } else {
+            Optional<Path> configDir = platformConfig.getConfigDir();
+            if (configDir.isPresent()) {
+                return configDir.get().resolve(fixesFolderName);
+            } else {
+                LOG.error("Executing {} pre processor. The folder name for the output of files containing required fixes is a relative path ({}), but the platform config dir is empty.", NAME, fixesFolderName);
+                return null;
+            }
+        }
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public void process(CgmesModel cgmes) {
+        Objects.requireNonNull(cgmes);
+        String basename = nameFor(cgmes);
+
+        Path fixesFolder = getFixesFolder();
+        if (fixesFolder != null) {
+            prepareAndReadFixesUsingFolder(cgmes, basename, fixesFolder);
+        }
+    }
+
+    private static class RegionContainers {
+        String subGeographicalRegionId;
+        String geographicalRegionId;
+    }
+}