Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CGMES export. Handle CGM export dependencies #2927

Merged
merged 35 commits into from
Jun 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
70bccec
CGMES export handles CGM export dependencies
zamarrenolm Mar 12, 2024
487490e
CGM SV export depends on IGM original TPs
zamarrenolm Mar 15, 2024
5c7cdc4
Add a new parameter and logic for CGM export
rcourtier Mar 22, 2024
4dae10f
Handling of metadata model is now done in the CgmesExport class
rcourtier Mar 27, 2024
c1888c3
Refactor export method to reduce its complexity
rcourtier Apr 4, 2024
e4af5e4
Reworked CGM export and added unit tests
rcourtier Apr 12, 2024
d8f3cba
Merge branch 'main' into cgmes_cgm_export_dependencies
zamarrenolm Apr 23, 2024
0d05b0e
merge main, solve conflicts
zamarrenolm Apr 25, 2024
39277ce
use booleans
zamarrenolm Apr 25, 2024
e255c01
merge main, solve conflicts
zamarrenolm May 3, 2024
d47d3a4
Allow external preparation of dependencies
zamarrenolm May 3, 2024
a1a30e9
add unit test for cgm export and update dependencies parameter
zamarrenolm May 3, 2024
2a042ce
Merge branch 'main' into cgmes_cgm_export_dependencies
annetill May 10, 2024
b91b911
Merge branch 'main' into cgmes_cgm_export_dependencies
zamarrenolm May 31, 2024
868dc4f
unit test to reproduce the farao use case
zamarrenolm May 31, 2024
f00b2cd
Merge branch 'main' into cgmes_cgm_export_dependencies
zamarrenolm Jun 4, 2024
b73f59b
Merge branch 'main' into cgmes_cgm_export_dependencies
zamarrenolm Jun 12, 2024
db7c287
Merge branch 'main' into cgmes_cgm_export_dependencies
zamarrenolm Jun 12, 2024
e358010
clean up unit tests
zamarrenolm Jun 12, 2024
e241306
documentation
zamarrenolm Jun 12, 2024
04db632
clarify handling of version
zamarrenolm Jun 13, 2024
16d41d3
remove empty line
zamarrenolm Jun 13, 2024
6f6ffe2
remove unused parameter name
zamarrenolm Jun 17, 2024
11f1175
generic notes for export summary and options
zamarrenolm Jun 17, 2024
2d0bfdf
Merge branch 'main' into cgmes_cgm_export_dependencies
zamarrenolm Jun 17, 2024
f1234c4
use given supersede information
zamarrenolm Jun 17, 2024
17f3381
updated documentation
zamarrenolm Jun 17, 2024
3b34c79
unit test for demonstrating manual cgm export
zamarrenolm Jun 17, 2024
2bdabca
Merge branch 'main' into cgmes_cgm_export_dependencies
zamarrenolm Jun 17, 2024
a6f7b6d
Merge branch 'main' into cgmes_cgm_export_dependencies
annetill Jun 18, 2024
0336c6f
clarify version number in CGM quick export
zamarrenolm Jun 18, 2024
825c764
Clean.
annetill Jun 18, 2024
2bf7181
clarify CGM quick export ignores profiles parameter always exports SS…
zamarrenolm Jun 18, 2024
82e16c0
Merge branch 'main' into cgmes_cgm_export_dependencies
annetill Jun 19, 2024
b3e0e2b
Merge branch 'main' into cgmes_cgm_export_dependencies
annetill Jun 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,14 @@
package com.powsybl.cgmes.completion;

import com.google.auto.service.AutoService;
import com.powsybl.cgmes.conversion.CgmesExport;
import com.powsybl.cgmes.conversion.CgmesImportPreProcessor;
import com.powsybl.cgmes.conversion.export.CgmesExportContext;
import com.powsybl.cgmes.conversion.export.CgmesExportUtil;
import com.powsybl.cgmes.conversion.export.elements.*;
import com.powsybl.cgmes.extensions.CgmesTopologyKind;
import com.powsybl.cgmes.extensions.CimCharacteristicsAdder;
import com.powsybl.cgmes.model.CgmesMetadataModel;
import com.powsybl.cgmes.model.CgmesModel;
import com.powsybl.cgmes.model.CgmesNames;
import com.powsybl.cgmes.model.CgmesSubset;
Expand Down Expand Up @@ -220,7 +222,9 @@ private static void writeHeader(Network network, XMLStreamWriter writer, CgmesEx
String euNamespace = context.getCim().getEuNamespace();
CgmesExportUtil.writeRdfRoot(cimNamespace, context.getCim().getEuPrefix(), euNamespace, writer);
if (context.getCimVersion() >= 16) {
CgmesExportUtil.writeModelDescription(network, CgmesSubset.EQUIPMENT, writer, context.getExportedEQModel(), context);
CgmesMetadataModel eqModel = CgmesExport.initializeModelForExport(
network, CgmesSubset.EQUIPMENT, context, true, false);
CgmesExportUtil.writeModelDescription(network, CgmesSubset.EQUIPMENT, writer, eqModel, context);
}
}

Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -51,22 +51,23 @@ public class CgmesExportContext {
private static final String REGION_NAME = "regionName";
private static final String DEFAULT_REGION = "default region";
public static final String SUB_REGION_ID = "subRegionId";
private static final String BOUNDARY_EQ_ID_PROPERTY = Conversion.CGMES_PREFIX_ALIAS_PROPERTIES + "EQ_BD_ID";
private static final String BOUNDARY_TP_ID_PROPERTY = Conversion.CGMES_PREFIX_ALIAS_PROPERTIES + "TP_BD_ID";

private CgmesNamespace.Cim cim = CgmesNamespace.CIM_16;
private CgmesTopologyKind topologyKind = CgmesTopologyKind.BUS_BRANCH;
private ZonedDateTime scenarioTime = ZonedDateTime.now();
private ReportNode reportNode = ReportNode.NO_OP;
private String boundaryEqId; // may be null
private String boundaryTpId; // may be null
private String businessProcess = DEFAULT_BUSINESS_PROCESS;

private final CgmesMetadataModel exportedEQModel = new CgmesMetadataModelImpl(CgmesSubset.EQUIPMENT, DEFAULT_MODELING_AUTHORITY_SET_VALUE);
private final CgmesMetadataModel exportedTPModel = new CgmesMetadataModelImpl(CgmesSubset.TOPOLOGY, DEFAULT_MODELING_AUTHORITY_SET_VALUE);
private final CgmesMetadataModel exportedSVModel = new CgmesMetadataModelImpl(CgmesSubset.STATE_VARIABLES, DEFAULT_MODELING_AUTHORITY_SET_VALUE);
private final CgmesMetadataModel exportedSSHModel = new CgmesMetadataModelImpl(CgmesSubset.STEADY_STATE_HYPOTHESIS, DEFAULT_MODELING_AUTHORITY_SET_VALUE);

private NamingStrategy namingStrategy = new NamingStrategy.Identity();

private String modelingAuthoritySet = null;
private String modelDescription = null;
private String modelVersion = null;
private String boundaryEqId = null;
private String boundaryTpId = null;
private List<String> profiles = null;
private String baseName = null;
public static final boolean CGM_EXPORT_VALUE = false;
public static final boolean EXPORT_BOUNDARY_POWER_FLOWS_DEFAULT_VALUE = true;
public static final boolean EXPORT_POWER_FLOWS_FOR_SWITCHES_DEFAULT_VALUE = true;
public static final boolean EXPORT_TRANSFORMERS_WITH_HIGHEST_VOLTAGE_AT_END1_DEFAULT_VALUE = false;
Expand Down Expand Up @@ -99,59 +100,6 @@ public class CgmesExportContext {
private final Map<String, String> fictitiousContainers = new HashMap<>();
private final Map<String, Bus> topologicalNodes = new HashMap<>();
private final ReferenceDataProvider referenceDataProvider;
private final EnumMap<CgmesSubset, List<String>> legacyIdsForSvDependencies = new EnumMap<>(CgmesSubset.class);

/**
* Update dependencies in a way that:
* SV depends on TP and SSH
* TP depends on EQ
* SSH depends on EQ
* If the boundaries subset have been defined:
* EQ depends on EQ_BD
* SV depends on TP_BD
*/
public void updateDependencies() {
if (!updateDependencies) {
return;
}
String eqModelId = getExportedEQModel().getId();
if (eqModelId == null || eqModelId.isEmpty()) {
return;
}

getExportedTPModel()
.clearDependencies()
.addDependentOn(eqModelId);

getExportedSSHModel()
.clearDependencies()
.addDependentOn(eqModelId);

getExportedSVModel().clearDependencies();
List<String> tpIds = legacyIdsForSvDependencies.get(CgmesSubset.TOPOLOGY);
if (tpIds != null) {
// If the list of SV dependencies from TP files has been set, even if it is empty,
// use it and ignore the exported TP model
getExportedSVModel().addDependentOn(tpIds);
} else {
getExportedSVModel().addDependentOn(getExportedTPModel().getId());
}
List<String> sshIds = legacyIdsForSvDependencies.get(CgmesSubset.STEADY_STATE_HYPOTHESIS);
if (sshIds != null) {
// If the list of SV dependencies from SSH files has been set, even if it is empty,
// use it and ignore the exported SSH model
getExportedSVModel().addDependentOn(sshIds);
} else {
getExportedSVModel().addDependentOn(getExportedSSHModel().getId());
}

if (boundaryEqId != null) {
getExportedEQModel().addDependentOn(boundaryEqId);
}
if (boundaryTpId != null) {
getExportedSVModel().addDependentOn(boundaryTpId);
}
}

public String getFictitiousContainerFor(Identifiable<?> id) {
return fictitiousContainers.get(id.getId());
Expand All @@ -162,7 +110,6 @@ public void setFictitiousContainerFor(Identifiable<?> id, String containerId) {
}

public CgmesExportContext() {
initializeExportedModelProfiles(this.cim);
referenceDataProvider = null;
}

Expand All @@ -183,7 +130,6 @@ public CgmesExportContext(Network network, ReferenceDataProvider referenceDataPr
}

public CgmesExportContext(Network network, ReferenceDataProvider referenceDataProvider, NamingStrategy namingStrategy) {
initializeExportedModelProfiles(this.cim);
this.referenceDataProvider = referenceDataProvider;
this.namingStrategy = namingStrategy;
CimCharacteristics cimCharacteristics = network.getExtension(CimCharacteristics.class);
Expand All @@ -194,41 +140,14 @@ public CgmesExportContext(Network network, ReferenceDataProvider referenceDataPr
topologyKind = networkTopologyKind(network);
}
scenarioTime = network.getCaseDate();
CgmesMetadataModels models = network.getExtension(CgmesMetadataModels.class);
if (models != null) {
models.getModelForSubset(CgmesSubset.EQUIPMENT).ifPresent(eq -> prepareExportedModelFrom(exportedEQModel, eq));
models.getModelForSubset(CgmesSubset.STEADY_STATE_HYPOTHESIS).ifPresent(ssh -> prepareExportedModelFrom(exportedSSHModel, ssh));
models.getModelForSubset(CgmesSubset.TOPOLOGY).ifPresent(tp -> prepareExportedModelFrom(exportedTPModel, tp));
models.getModelForSubset(CgmesSubset.STATE_VARIABLES).ifPresent(sv -> prepareExportedModelFrom(exportedSVModel, sv));
}
addIidmMappings(network);
}

/**
* Set for each exported model the profile relative to the cim version.
* @param cim The cim version from which depends the models profiles uri.
*/
private void initializeExportedModelProfiles(CgmesNamespace.Cim cim) {
if (cim.hasProfiles()) {
exportedEQModel.setProfile(cim.getProfileUri("EQ"));
exportedTPModel.setProfile(cim.getProfileUri("TP"));
exportedSVModel.setProfile(cim.getProfileUri("SV"));
exportedSSHModel.setProfile(cim.getProfileUri("SSH"));
if (network.hasProperty(BOUNDARY_EQ_ID_PROPERTY)) {
setBoundaryEqId(network.getProperty(BOUNDARY_EQ_ID_PROPERTY));
}
if (network.hasProperty(BOUNDARY_TP_ID_PROPERTY)) {
setBoundaryTpId(network.getProperty(BOUNDARY_TP_ID_PROPERTY));
}
}

/**
* Update the model used for the export according to the network model.
* All the metadata information will be duplicated, except for the version that will be incremented.
* @param exportedModel The {@link CgmesMetadataModel} used for the export.
* @param fromModel The {@link CgmesMetadataModel} attached to the network.
*/
private void prepareExportedModelFrom(CgmesMetadataModel exportedModel, CgmesMetadataModel fromModel) {
exportedModel.setDescription(fromModel.getDescription());
exportedModel.setVersion(fromModel.getVersion() + 1);
Copy link
Contributor

@WalAmeni WalAmeni May 21, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The exported model does no longer increment the version for SSH and SV when they are generated separately

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The metadata models in the Network are never updated after export (this is by design). To obtain the version of the exported files one must read the contents of the output.

I've prepared a unit test that I think demonstrates and clarifies how this CGM export can be used. It is inside CommonGridModelExportTest::testFaraoUseCase.

exportedModel.addSupersedes(fromModel.getId());
Copy link
Contributor

@WalAmeni WalAmeni May 21, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The md:Model.Supersedes is no longer exported in the SSH and SV file when they are generated separately

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same comment as above. Please review the added unit test. The supersedes information must also be obtained from the output file.

exportedModel.addDependentOn(fromModel.getDependentOn());
exportedModel.setModelingAuthoritySet(fromModel.getModelingAuthoritySet());
}

private CgmesTopologyKind networkTopologyKind(Network network) {
Expand Down Expand Up @@ -367,16 +286,6 @@ public boolean isExportedEquipment(Identifiable<?> c) {
return !ignored;
}

public CgmesExportContext setBoundaryEqId(String boundaryEqId) {
this.boundaryEqId = boundaryEqId;
return this;
}

public CgmesExportContext setBoundaryTpId(String boundaryTpId) {
this.boundaryTpId = boundaryTpId;
return this;
}

private void addIidmMappingsSwitchTerminals(Network network) {
for (Switch sw : network.getSwitches()) {
String terminal1Id = sw.getAliasFromType(Conversion.CGMES_PREFIX_ALIAS_PROPERTIES + CgmesNames.TERMINAL + "1").orElse(null);
Expand Down Expand Up @@ -637,7 +546,6 @@ public int getCimVersion() {

public CgmesExportContext setCimVersion(int cimVersion) {
cim = CgmesNamespace.getCim(cimVersion);
initializeExportedModelProfiles(cim);
return this;
}

Expand All @@ -659,22 +567,6 @@ public CgmesExportContext setScenarioTime(ZonedDateTime scenarioTime) {
return this;
}

public CgmesMetadataModel getExportedEQModel() {
return exportedEQModel;
}

public CgmesMetadataModel getExportedTPModel() {
return exportedTPModel;
}

public CgmesMetadataModel getExportedSVModel() {
return exportedSVModel;
}

public CgmesMetadataModel getExportedSSHModel() {
return exportedSSHModel;
}

public boolean exportBoundaryPowerFlows() {
return exportBoundaryPowerFlows;
}
Expand Down Expand Up @@ -823,13 +715,76 @@ public CgmesExportContext setBusinessProcess(String businessProcess) {
return this;
}

public void setLegacyIdsForSvDependencies(CgmesSubset subset, List<String> ids) {
legacyIdsForSvDependencies.put(subset, ids);
public String getModelingAuthoritySet() {
return modelingAuthoritySet;
}

public CgmesExportContext setModelingAuthoritySet(String modelingAuthoritySet) {
this.modelingAuthoritySet = modelingAuthoritySet;
return this;
}

public String getModelDescription() {
return modelDescription;
}

public CgmesExportContext setModelDescription(String modelDescription) {
this.modelDescription = modelDescription;
return this;
}

public String getModelVersion() {
return modelVersion;
}

public CgmesExportContext setModelVersion(String modelVersion) {
this.modelVersion = modelVersion;
return this;
}

public String getBoundaryEqId() {
return boundaryEqId;
}

public CgmesExportContext setBoundaryEqId(String boundaryEqId) {
this.boundaryEqId = boundaryEqId;
return this;
}

public String getBoundaryTpId() {
return boundaryTpId;
}

public CgmesExportContext setBoundaryTpId(String boundaryTpId) {
this.boundaryTpId = boundaryTpId;
return this;
}

public List<String> getProfiles() {
return profiles;
}

public CgmesExportContext setProfiles(List<String> profiles) {
this.profiles = profiles;
return this;
}

public String getBaseName() {
return baseName;
}

public CgmesExportContext setBaseName(String baseName) {
this.baseName = baseName;
return this;
}

public CgmesExportContext setUpdateDependencies(boolean updateDependencies) {
this.updateDependencies = updateDependencies;
return this;
}

public boolean updateDependencies() {
return updateDependencies;
}
}

Original file line number Diff line number Diff line change
Expand Up @@ -116,24 +116,28 @@ public static void writeRdfRoot(String cimNamespace, String euPrefix, String euN
writer.writeNamespace("md", MD_NAMESPACE);
}

public static void writeModelDescription(Network network, CgmesSubset subset, XMLStreamWriter writer, CgmesMetadataModel modelDescription, CgmesExportContext context) throws XMLStreamException {
public static void initializeModelId(Network network, CgmesMetadataModel model, CgmesExportContext context) {
// The ref to build a unique model id must contain:
// the network, the subset (EQ, SSH, SV, ...), the time of the scenario, the version, the business process and the FULL_MODEL part
// If we use name-based UUIDs this ensures that the UUID for the model will be specific enough
CgmesObjectReference[] modelRef = {
refTyped(network),
ref(subset),
ref(model.getSubset()),
ref(DATE_TIME_FORMATTER.format(context.getScenarioTime())),
ref(format(modelDescription.getVersion())),
ref(String.valueOf(model.getVersion())),
ref(context.getBusinessProcess()),
Part.FULL_MODEL};
String modelId = "urn:uuid:" + context.getNamingStrategy().getCgmesId(modelRef);
modelDescription.setId(modelId);
context.updateDependencies();
model.setId(modelId);
}

public static void writeModelDescription(Network network, CgmesSubset subset, XMLStreamWriter writer, CgmesMetadataModel modelDescription, CgmesExportContext context) throws XMLStreamException {
if (modelDescription.getId() == null || modelDescription.getId().isEmpty()) {
initializeModelId(network, modelDescription, context);
}
writer.writeStartElement(MD_NAMESPACE, "FullModel");
writer.writeAttribute(RDF_NAMESPACE, CgmesNames.ABOUT, modelId);
context.getReportNode().newReportNode().withMessageTemplate("CgmesId", modelId).add();
writer.writeAttribute(RDF_NAMESPACE, CgmesNames.ABOUT, modelDescription.getId());
context.getReportNode().newReportNode().withMessageTemplate("CgmesId", modelDescription.getId()).add();
writer.writeStartElement(MD_NAMESPACE, CgmesNames.SCENARIO_TIME);
writer.writeCharacters(DATE_TIME_FORMATTER.format(context.getScenarioTime()));
writer.writeEndElement();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,13 @@
*/
package com.powsybl.cgmes.conversion.export;

import com.powsybl.cgmes.conversion.CgmesExport;
import com.powsybl.cgmes.conversion.Conversion;
import com.powsybl.cgmes.conversion.naming.CgmesObjectReference;
import com.powsybl.cgmes.conversion.naming.NamingStrategy;
import com.powsybl.cgmes.conversion.export.elements.*;
import com.powsybl.cgmes.extensions.*;
import com.powsybl.cgmes.model.CgmesMetadataModel;
import com.powsybl.cgmes.model.CgmesNames;
import com.powsybl.cgmes.model.CgmesSubset;
import com.powsybl.commons.PowsyblException;
Expand Down Expand Up @@ -54,7 +56,12 @@ public static void write(Network network, XMLStreamWriter writer) {
}

public static void write(Network network, XMLStreamWriter writer, CgmesExportContext context) {
context.setExportEquipment(true);
CgmesMetadataModel model = CgmesExport.initializeModelForExport(
network, CgmesSubset.EQUIPMENT, context, true, false);
write(network, writer, context, model);
}

public static void write(Network network, XMLStreamWriter writer, CgmesExportContext context, CgmesMetadataModel model) {
try {
boolean writeConnectivityNodes = context.writeConnectivityNodes();

Expand All @@ -68,7 +75,7 @@ public static void write(Network network, XMLStreamWriter writer, CgmesExportCon
CgmesExportUtil.writeRdfRoot(cimNamespace, context.getCim().getEuPrefix(), euNamespace, writer);

if (context.getCimVersion() >= 16) {
CgmesExportUtil.writeModelDescription(network, CgmesSubset.EQUIPMENT, writer, context.getExportedEQModel(), context);
CgmesExportUtil.writeModelDescription(network, CgmesSubset.EQUIPMENT, writer, model, context);
}

Map<String, String> mapNodeKey2NodeId = new HashMap<>();
Expand Down
Loading