Skip to content

Commit

Permalink
Merge branch 'main' into write-properties-for-calculated-buses
Browse files Browse the repository at this point in the history
  • Loading branch information
@EtienneLt
EtienneLt authored Jun 29, 2023
2 parents 0558828 + c2ccb73 commit a63ee17
Show file tree
Hide file tree
Showing 82 changed files with 5,670 additions and 618 deletions.
103 changes: 103 additions & 0 deletions cgmes/cgmes-completion/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (c) 2023, RTE (http://www.rte-france.com)
This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.
SPDX-License-Identifier: MPL-2.0
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>com.powsybl</groupId>
<artifactId>powsybl-cgmes</artifactId>
<version>5.4.0-SNAPSHOT</version>
</parent>

<artifactId>powsybl-cgmes-completion</artifactId>
<name>CGMES network model completion</name>
<description>CGMES (Common Grid Model Exchange Specification). Complete missing data in input instance files
</description>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifestEntries>
<Automatic-Module-Name>com.powsybl.cgmes.completion</Automatic-Module-Name>
</manifestEntries>
</archive>
</configuration>
</plugin>
</plugins>
</build>

<dependencies>
<!-- Compilation dependencies -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>powsybl-cgmes-conversion</artifactId>
<version>${project.version}</version>
</dependency>

<!-- Test dependencies -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.jimfs</groupId>
<artifactId>jimfs</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>powsybl-config-test</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>powsybl-cgmes-conformity</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>powsybl-iidm-impl</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>

<!-- Test dependencies include all considered triple store engines -->
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>powsybl-triple-store-impl-rdf4j</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
263 changes: 263 additions & 0 deletions ...etion/src/main/java/com/powsybl/cgmes/completion/CreateMissingContainersPreProcessor.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,263 @@
/**
* Copyright (c) 2023, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
* SPDX-License-Identifier: MPL-2.0
*/
package com.powsybl.cgmes.completion;

import com.google.auto.service.AutoService;
import com.powsybl.cgmes.conversion.CgmesImportPreProcessor;
import com.powsybl.cgmes.conversion.export.CgmesExportContext;
import com.powsybl.cgmes.conversion.export.CgmesExportUtil;
import com.powsybl.cgmes.conversion.export.elements.*;
import com.powsybl.cgmes.extensions.CgmesTopologyKind;
import com.powsybl.cgmes.extensions.CimCharacteristicsAdder;
import com.powsybl.cgmes.model.CgmesModel;
import com.powsybl.cgmes.model.CgmesNames;
import com.powsybl.cgmes.model.triplestore.CgmesModelTripleStore;
import com.powsybl.commons.PowsyblException;
import com.powsybl.commons.config.PlatformConfig;
import com.powsybl.commons.datasource.ZipFileDataSource;
import com.powsybl.commons.parameters.Parameter;
import com.powsybl.commons.parameters.ParameterDefaultValueConfig;
import com.powsybl.commons.parameters.ParameterType;
import com.powsybl.commons.reporter.Reporter;
import com.powsybl.commons.xml.XmlUtil;
import com.powsybl.iidm.network.Network;
import com.powsybl.iidm.network.NetworkFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

/**
* <p>
* A CGMES pre-processor that defines missing containers in input data.
* </p>
*
* <p>
* The pre-processor will analyze the input data and check if there are missing container definitions.
* It will then create a CIM-XML file with all required objects (voltage levels, substations, regions, ...)
* that will be used during CGMES import to allow the conversion to PowSyBl Network.
* </p>
*
* <p>
* It is assumed that all containers missing are voltage levels.
* The user can specify the location folder of the output files using the parameter <code>iidm.import.cgmes.fixes-for-missing-containers-folder</code>.
* The CIM version of the output file will be the same detected for the input data.
* Because no information about voltage level is available, a default arbitrary value for nominal voltage is used.
* The user may edit the generated files and reuse them in successive imports.
* </p>
*
* @author Luma Zamarreño <zamarrenolm at aia.es>
*/
@AutoService(CgmesImportPreProcessor.class)
public class CreateMissingContainersPreProcessor implements CgmesImportPreProcessor {

public static final String NAME = "createMissingContainers";
public static final String FIXES_FOLDER_NAME = "iidm.import.cgmes.fixes-for-missing-containers-folder";
public static final double DEFAULT_NOMINAL_VALUE_FOR_MISSING_VOLTAGE_LEVELS = 100.0;

private static final Logger LOG = LoggerFactory.getLogger(CreateMissingContainersPreProcessor.class);
private static final Parameter FIXES_FOLDER_NAME_PARAMETER = new Parameter(FIXES_FOLDER_NAME,
ParameterType.STRING,
"Folder where zip files containing fixes will be created: one zip for each imported network missing data",
null);

private final PlatformConfig platformConfig;
private final ParameterDefaultValueConfig defaultValueConfig;

public CreateMissingContainersPreProcessor(PlatformConfig platformConfig) {
Objects.requireNonNull(platformConfig);
this.platformConfig = platformConfig;
defaultValueConfig = new ParameterDefaultValueConfig(platformConfig);
}

public CreateMissingContainersPreProcessor() {
this(PlatformConfig.defaultConfig());
}

private static String nameFor(CgmesModel cgmes) {
try {
return new URI(cgmes.getBasename()).getAuthority();
} catch (URISyntaxException e) {
return cgmes.modelId();
}
}

private static void prepareAndReadFixesUsingFolder(CgmesModel cgmes, String basename, Path fixesFolder) {
if (!Files.isDirectory(fixesFolder)) {
LOG.error("Output folder is not a directory {}. Skipping post processor.", fixesFolder);
return;
}
Path fixesFile = fixesFolder.resolve(basename + ".zip");
// Check the file will be writable
try {
Files.deleteIfExists(fixesFile);
Files.createFile(fixesFile);
} catch (IOException e) {
LOG.error("Output file {} is not writable. Skipping post processor.", fixesFile);
return;
}
if (LOG.isInfoEnabled()) {
LOG.info("Execute {} pre processor on CGMES model {}. Output to file {}", NAME, cgmes.modelId(), fixesFile);
}
prepareAndReadFixesUsingZipFile(cgmes, basename, fixesFile);
}

private static void prepareAndReadFixesUsingZipFile(CgmesModel cgmes, String basename, Path fixesFile) {
// Assume all containers missing are voltage levels and create proper objects for them (substation, regions, ...)
Set<String> missingVoltageLevels = findMissingVoltageLevels(cgmes);
LOG.info("Missing voltage levels: {}", missingVoltageLevels);
if (!missingVoltageLevels.isEmpty()) {
buildZipFileWithFixes(cgmes, missingVoltageLevels, fixesFile, basename);
cgmes.read(new ZipFileDataSource(fixesFile), Reporter.NO_OP);
}
Set<String> missingVoltageLevelsAfterFix = findMissingVoltageLevels(cgmes);
if (!missingVoltageLevelsAfterFix.isEmpty()) {
throw new IllegalStateException("Missing voltage levels after fix: " + missingVoltageLevelsAfterFix);
}
// The only containers without voltage level must be of type line
LOG.info("After the fixes have been applied, the only node containers without voltage level must be of type Line.");
LOG.info("Containers without voltage level that are not Lines will be reported as errors.");
cgmes.connectivityNodeContainers().stream()
.filter(c -> c.getId(CgmesNames.VOLTAGE_LEVEL) == null)
.filter(c -> !c.getLocal("connectivityNodeContainerType").equals("Line"))
.forEach(c -> LOG.error(c.getId(CgmesNames.CONNECTIVITY_NODE_CONTAINER)));
}

private static Set<String> findMissingVoltageLevels(CgmesModel cgmes) {
// check missing CN containers
Set<String> defined = cgmes.connectivityNodeContainers().stream().map(c -> c.getId(CgmesNames.CONNECTIVITY_NODE_CONTAINER)).collect(Collectors.toSet());
Set<String> referred = cgmes.connectivityNodes().stream().map(c -> c.getId(CgmesNames.CONNECTIVITY_NODE_CONTAINER)).collect(Collectors.toSet());
return referred.stream().filter(c -> !defined.contains(c)).collect(Collectors.toSet());
}

private static void buildZipFileWithFixes(CgmesModel cgmes, Set<String> missingVoltageLevels, Path fixesFile, String basename) {
Network network = prepareEmptyNetworkForExport(cgmes);
CgmesExportContext context = new CgmesExportContext(network);
try (ZipOutputStream zout = new ZipOutputStream(Files.newOutputStream(fixesFile))) {
zout.putNextEntry(new ZipEntry(basename + "_EQ.xml"));
XMLStreamWriter writer = XmlUtil.initializeWriter(true, " ", zout);
writeHeader(writer, context);
RegionContainers regionContainers = writeRegionContainers(writer, context);
for (String missingVoltageLevel : missingVoltageLevels) {
writeMissingVoltageLevel(missingVoltageLevel, writer, context, regionContainers);
}
writer.writeEndDocument();
zout.closeEntry();
} catch (IOException | XMLStreamException x) {
throw new PowsyblException("Building file containing fixes for missing data", x);
}
}

private static Network prepareEmptyNetworkForExport(CgmesModel cgmes) {
Network network = NetworkFactory.findDefault().createNetwork("empty", "CGMES");
// We ensure that the fixes are exported to CGMES files with the same version of the input files
// To achieve it, we set the CIM characteristics of the empty Network created
if (cgmes instanceof CgmesModelTripleStore) {
network.newExtension(CimCharacteristicsAdder.class)
.setTopologyKind(cgmes.isNodeBreaker() ? CgmesTopologyKind.NODE_BREAKER : CgmesTopologyKind.BUS_BRANCH)
.setCimVersion(((CgmesModelTripleStore) cgmes).getCimVersion())
.add();
}
return network;
}

private static RegionContainers writeRegionContainers(XMLStreamWriter writer, CgmesExportContext context) throws XMLStreamException {
String cimNamespace = context.getCim().getNamespace();

// An alternative to replicate this code would be to make public the method
// EquipmentExport::writeFictitiousSubstationFor and use it here.
// We could group all missing voltage levels in the same (fictitious) substation
RegionContainers regionContainers = new RegionContainers();
regionContainers.subGeographicalRegionId = CgmesExportUtil.getUniqueId();
String subGeographicalRegionName = "SGR fix for missing data";
regionContainers.geographicalRegionId = CgmesExportUtil.getUniqueId();
String geographicalRegionName = "GR fix for missing data";
SubGeographicalRegionEq.write(regionContainers.subGeographicalRegionId, subGeographicalRegionName, regionContainers.geographicalRegionId, cimNamespace, writer, context);
GeographicalRegionEq.write(regionContainers.geographicalRegionId, geographicalRegionName, cimNamespace, writer, context);
return regionContainers;
}

private static void writeMissingVoltageLevel(String voltageLevelId, XMLStreamWriter writer, CgmesExportContext context, RegionContainers regionContainers) throws XMLStreamException {
String cimNamespace = context.getCim().getNamespace();

// In a first approach,
// we do not have additional information about the voltage level,
// we create a different substation and base voltage for every missing voltage level
String voltageLevelName = voltageLevelId + " VL";
String substationId = CgmesExportUtil.getUniqueId();
String substationName = voltageLevelId + "SUB for missing VL " + voltageLevelId;
String baseVoltageId = CgmesExportUtil.getUniqueId();

VoltageLevelEq.write(voltageLevelId, voltageLevelName, Double.NaN, Double.NaN, substationId, baseVoltageId, cimNamespace, writer, context);
SubstationEq.write(substationId, substationName, regionContainers.subGeographicalRegionId, cimNamespace, writer, context);
BaseVoltageEq.write(baseVoltageId, DEFAULT_NOMINAL_VALUE_FOR_MISSING_VOLTAGE_LEVELS, cimNamespace, writer, context);
}

private static void writeHeader(XMLStreamWriter writer, CgmesExportContext context) throws XMLStreamException {
String cimNamespace = context.getCim().getNamespace();
String euNamespace = context.getCim().getEuNamespace();
CgmesExportUtil.writeRdfRoot(cimNamespace, context.getCim().getEuPrefix(), euNamespace, writer);
if (context.getCimVersion() >= 16) {
ModelDescriptionEq.write(writer, context.getEqModelDescription(), context);
}
}

private Path getFixesFolder() {
String fixesFolderName = Parameter.readString("CGMES", null, FIXES_FOLDER_NAME_PARAMETER, defaultValueConfig);
if (fixesFolderName == null) {
LOG.error("Executing {} pre processor. Missing the folder name for the output of files containing required fixes. Use the parameter {}.", NAME, FIXES_FOLDER_NAME_PARAMETER.getName());
return null;
}
Path fixesFolder = Paths.get(fixesFolderName);
if (fixesFolder.isAbsolute()) {
return fixesFolder;
} else {
Optional<Path> configDir = platformConfig.getConfigDir();
if (configDir.isPresent()) {
return configDir.get().resolve(fixesFolderName);
} else {
LOG.error("Executing {} pre processor. The folder name for the output of files containing required fixes is a relative path ({}), but the platform config dir is empty.", NAME, fixesFolderName);
return null;
}
}
}

@Override
public String getName() {
return NAME;
}

@Override
public void process(CgmesModel cgmes) {
Objects.requireNonNull(cgmes);
String basename = nameFor(cgmes);

Path fixesFolder = getFixesFolder();
if (fixesFolder != null) {
prepareAndReadFixesUsingFolder(cgmes, basename, fixesFolder);
}
}

private static class RegionContainers {
String subGeographicalRegionId;
String geographicalRegionId;
}
}
Loading

0 comments on commit a63ee17

Please sign in to comment.