Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Datasources - Part 1: Rework datasource classes #3101

Merged
merged 9 commits into from
Jul 19, 2024
Merged

Datasources - Part 1: Rework datasource classes #3101

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
e5c4154
reorganize datasource classes + add tests
rolnico Jul 10, 2024
f7be34e
fix issues + make a constructor protected
rolnico Jul 11, 2024
2a59770
move boolean exists(String suffix, String ext) to AbstractFileSystemD…
rolnico Jul 15, 2024
5db7158
remove mention to mainExtension in tests
rolnico Jul 15, 2024
77b606f
remove semicolon
rolnico Jul 15, 2024
76c5981
factorize identical sets
rolnico Jul 19, 2024
a10185c
fix tests
rolnico Jul 19, 2024
1d388c9
fix issue
rolnico Jul 19, 2024
59b687d
Merge branch 'main' into nro/datasources_reorganisation
flo-dup Jul 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ampl-executor/src/main/java/com/powsybl/ampl/executor/AmplModelExecutionHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import com.powsybl.ampl.converter.*;
import com.powsybl.commons.PowsyblException;
import com.powsybl.commons.datasource.DataSource;
import com.powsybl.commons.datasource.FileDataSource;
import com.powsybl.commons.datasource.DirectoryDataSource;
import com.powsybl.commons.util.StringToIntMapper;
import com.powsybl.computation.*;
import com.powsybl.iidm.network.Network;
Expand Down Expand Up @@ -97,7 +97,7 @@ private void exportAmplParameters(Path workingDir) throws IOException {
}

private void exportNetworkAsAmpl(Path workingDir) {
DataSource networkExportDataSource = new FileDataSource(workingDir, this.model.getNetworkDataPrefix());
DataSource networkExportDataSource = new DirectoryDataSource(workingDir, this.model.getNetworkDataPrefix());
if (parameters.getAmplExportConfig() != null) {
new AmplExporter().export(network, parameters.getAmplExportConfig(), networkExportDataSource);
} else {
Expand Down Expand Up @@ -180,7 +180,7 @@ public List<CommandExecution> before(Path workingDir) throws IOException {
@Override
public AmplResults after(Path workingDir, ExecutionReport report) throws IOException {
super.after(workingDir.toAbsolutePath(), report);
DataSource networkAmplResults = new FileDataSource(workingDir, this.model.getOutputFilePrefix());
DataSource networkAmplResults = new DirectoryDataSource(workingDir, this.model.getOutputFilePrefix());
AmplNetworkReader reader = new AmplNetworkReader(networkAmplResults, this.network, this.model.getVariant(),
mapper, this.model.getNetworkUpdaterFactory(), this.model.getOutputFormat());
Map<String, String> indicators = readIndicators(reader);
Expand Down
4 changes: 2 additions & 2 deletions ...etion/src/main/java/com/powsybl/cgmes/completion/CreateMissingContainersPreProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import com.powsybl.cgmes.model.triplestore.CgmesModelTripleStore;
import com.powsybl.commons.PowsyblException;
import com.powsybl.commons.config.PlatformConfig;
import com.powsybl.commons.datasource.ZipFileDataSource;
import com.powsybl.commons.datasource.ZipArchiveDataSource;
import com.powsybl.commons.parameters.Parameter;
import com.powsybl.commons.parameters.ParameterDefaultValueConfig;
import com.powsybl.commons.parameters.ParameterType;
Expand Down Expand Up @@ -132,7 +132,7 @@ private static void prepareAndReadFixesUsingZipFile(CgmesModel cgmes, String bas
LOG.info("Missing voltage levels: {}", missingVoltageLevels);
if (!missingVoltageLevels.isEmpty()) {
buildZipFileWithFixes(cgmes, missingVoltageLevels, fixesFile, basename);
cgmes.read(new ZipFileDataSource(fixesFile), ReportNode.NO_OP);
cgmes.read(new ZipArchiveDataSource(fixesFile), ReportNode.NO_OP);
}
Set<String> missingVoltageLevelsAfterFix = findMissingVoltageLevels(cgmes);
if (!missingVoltageLevelsAfterFix.isEmpty()) {
Expand Down
12 changes: 6 additions & 6 deletions cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/ConversionTester.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@
import com.powsybl.cgmes.conversion.test.network.compare.ComparisonConfig;
import com.powsybl.cgmes.model.*;
import com.powsybl.commons.datasource.DataSource;
import com.powsybl.commons.datasource.FileDataSource;
import com.powsybl.commons.datasource.DirectoryDataSource;
import com.powsybl.commons.datasource.ReadOnlyDataSource;
import com.powsybl.commons.datasource.ZipFileDataSource;
import com.powsybl.commons.datasource.ZipArchiveDataSource;
import com.powsybl.iidm.network.Network;
import com.powsybl.iidm.network.impl.NetworkFactoryImpl;
import com.powsybl.iidm.serde.XMLExporter;
Expand Down Expand Up @@ -165,17 +165,17 @@ private static void exportXiidm(String name, String impl, Network expected, Netw
XMLExporter xmlExporter = new XMLExporter();
// Last component of the path is the name for the exported XML
if (expected != null) {
xmlExporter.export(expected, null, new FileDataSource(path, "expected"));
xmlExporter.export(expected, null, new DirectoryDataSource(path, "expected"));
}
if (actual != null) {
xmlExporter.export(actual, null, new FileDataSource(path, "actual"));
xmlExporter.export(actual, null, new DirectoryDataSource(path, "actual"));
}
}

private static void exportCgmes(String name, String impl, Network network) throws IOException {
String name1 = name.replace('/', '-');
Path path = Files.createTempDirectory("temp-export-cgmes-" + name1 + "-" + impl + "-");
new CgmesExport().export(network, null, new FileDataSource(path, "foo"));
new CgmesExport().export(network, null, new DirectoryDataSource(path, "foo"));
}

private static String subsetFromName(String name) {
Expand All @@ -197,7 +197,7 @@ private void testExportImportCgmes(Network network, ReadOnlyDataSource originalD
Path path = fs.getPath("temp-export-cgmes");
Files.createDirectories(path);
String baseName = originalDs.getBaseName();
DataSource ds = new ZipFileDataSource(path, baseName);
DataSource ds = new ZipArchiveDataSource(path, baseName);

// Copy the original files to the temporary destination, ensuring a normalized name
for (String name : new CgmesOnDataSource(originalDs).names()) {
Expand Down
4 changes: 2 additions & 2 deletions ...nversion/src/test/java/com/powsybl/cgmes/conversion/test/ImportExportPerformanceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import com.powsybl.cgmes.model.GridModelReference;
import com.powsybl.cgmes.model.test.Cim14SmallCasesCatalog;
import com.powsybl.commons.datasource.DataSource;
import com.powsybl.commons.datasource.FileDataSource;
import com.powsybl.commons.datasource.DirectoryDataSource;
import com.powsybl.commons.datasource.ReadOnlyDataSource;
import com.powsybl.iidm.network.Network;
import com.powsybl.iidm.network.NetworkFactory;
Expand Down Expand Up @@ -81,7 +81,7 @@ private void importExport(String ts, ReadOnlyDataSource ds, FileSystem fs) throw
CgmesExport e = new CgmesExport();
Path exportFolder = fs.getPath("impl-" + ts);
Files.createDirectories(exportFolder);
DataSource exportDataSource = new FileDataSource(exportFolder, "");
DataSource exportDataSource = new DirectoryDataSource(exportFolder, "");
Properties exportParameters = new Properties();
exportParameters.put(CgmesExport.CIM_VERSION, "16");
e.export(n, exportParameters, exportDataSource);
Expand Down
14 changes: 7 additions & 7 deletions ...es-conversion/src/test/java/com/powsybl/cgmes/conversion/test/export/CgmesExportTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -477,7 +477,7 @@ void testModelDescription() throws IOException {

try (FileSystem fileSystem = Jimfs.newFileSystem(Configuration.unix())) {
Path tmpDir = Files.createDirectory(fileSystem.getPath("tmp"));
ZipFileDataSource zip = new ZipFileDataSource(tmpDir.resolve("."), "output");
ZipArchiveDataSource zip = new ZipArchiveDataSource(tmpDir.resolve("."), "output");
new CgmesExport().export(network, params, zip);
Network network2 = Network.read(new GenericReadOnlyDataSource(tmpDir.resolve("output.zip")), importParams);
CgmesMetadataModel sshMetadata = network2
Expand All @@ -497,7 +497,7 @@ void testModelVersion() throws IOException {

try (FileSystem fileSystem = Jimfs.newFileSystem(Configuration.unix())) {
Path tmpDir = Files.createDirectory(fileSystem.getPath("tmp"));
ZipFileDataSource zip = new ZipFileDataSource(tmpDir.resolve("."), "output");
ZipArchiveDataSource zip = new ZipArchiveDataSource(tmpDir.resolve("."), "output");
new CgmesExport().export(network, params, zip);
Network network2 = Network.read(new GenericReadOnlyDataSource(tmpDir.resolve("output.zip")), importParams);
CgmesMetadataModel sshMetadata = network2.getExtension(CgmesMetadataModels.class).getModelForSubset(CgmesSubset.STEADY_STATE_HYPOTHESIS).orElseThrow();
Expand All @@ -519,7 +519,7 @@ void testModelDescriptionClosingXML() throws IOException {

try (FileSystem fileSystem = Jimfs.newFileSystem(Configuration.unix())) {
Path tmpDir = Files.createDirectory(fileSystem.getPath("tmp"));
ZipFileDataSource zip = new ZipFileDataSource(tmpDir.resolve("."), "output");
ZipArchiveDataSource zip = new ZipArchiveDataSource(tmpDir.resolve("."), "output");
new CgmesExport().export(network, params, zip);

// check network can be reimported and that ModelDescription still includes end-tag
Expand Down Expand Up @@ -561,7 +561,7 @@ void testExportWithModelingAuthorityFromReferenceData() throws IOException {
Properties exportParams = new Properties();
// It is enough to check that the MAS has been set correctly in the EQ instance file
exportParams.put(CgmesExport.PROFILES, "EQ");
new CgmesExport(platformConfig).export(network, exportParams, new FileDataSource(tmpDir, network.getNameOrId()));
new CgmesExport(platformConfig).export(network, exportParams, new DirectoryDataSource(tmpDir, network.getNameOrId()));

String eq = Files.readString(tmpDir.resolve(network.getNameOrId() + "_EQ.xml"));
assertTrue(eq.contains("modelingAuthoritySet>http://www.elia.be/OperationalPlanning"));
Expand Down Expand Up @@ -625,7 +625,7 @@ void testCanGeneratorControl() throws IOException {
Properties exportParams = new Properties();
exportParams.put(CgmesExport.PROFILES, "EQ");
// network.write("CGMES", null, tmpDir.resolve(baseName));
new CgmesExport().export(network, exportParams, new FileDataSource(tmpDir, baseName));
new CgmesExport().export(network, exportParams, new DirectoryDataSource(tmpDir, baseName));
String eq = Files.readString(tmpDir.resolve(baseName + "_EQ.xml"));

// Check that RC are exported properly
Expand All @@ -644,7 +644,7 @@ void testCanGeneratorControl() throws IOException {
generatorNoRcc.setTargetV(Double.NaN);

//network.write("CGMES", null, tmpDir.resolve(baseName));
new CgmesExport().export(network, exportParams, new FileDataSource(tmpDir, baseName));
new CgmesExport().export(network, exportParams, new DirectoryDataSource(tmpDir, baseName));
eq = Files.readString(tmpDir.resolve(baseName + "_EQ.xml"));
assertFalse(eq.contains("3a3b27be-b18b-4385-b557-6735d733baf0_RC"));
assertFalse(eq.contains("550ebe0d-f2b2-48c1-991f-cebea43a21aa_RC"));
Expand All @@ -665,7 +665,7 @@ void testCanGeneratorControl() throws IOException {
mmrlAdder.setMaxQ(mmrl.getMinQ());
mmrlAdder.add();

new CgmesExport().export(network, exportParams, new FileDataSource(tmpDir, baseName));
new CgmesExport().export(network, exportParams, new DirectoryDataSource(tmpDir, baseName));
eq = Files.readString(tmpDir.resolve(baseName + "_EQ.xml"));
assertFalse(eq.contains("3a3b27be-b18b-4385-b557-6735d733baf0_RC"));
assertFalse(eq.contains("550ebe0d-f2b2-48c1-991f-cebea43a21aa_RC"));
Expand Down
2 changes: 1 addition & 1 deletion ...rsion/src/test/java/com/powsybl/cgmes/conversion/test/export/CgmesNamingStrategyTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ private DataSource tmpDataSource(String folder, String baseName) throws IOExcept
FileUtils.cleanDirectory(exportFolder.toFile());
}
Files.createDirectories(exportFolder);
return new FileDataSource(exportFolder, baseName);
return new DirectoryDataSource(exportFolder, baseName);
}

private void copyBoundary(String outputFolderName, String baseName, ReadOnlyDataSource originalDataSource) throws IOException {
Expand Down
6 changes: 3 additions & 3 deletions ...ion/src/test/java/com/powsybl/cgmes/conversion/test/export/CommonGridModelExportTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import com.powsybl.cgmes.model.CgmesSubset;
import com.powsybl.commons.PowsyblException;
import com.powsybl.commons.datasource.DataSource;
import com.powsybl.commons.datasource.FileDataSource;
import com.powsybl.commons.datasource.DirectoryDataSource;
import com.powsybl.commons.datasource.MemDataSource;
import com.powsybl.commons.datasource.ReadOnlyDataSource;
import com.powsybl.commons.report.ReportNode;
Expand Down Expand Up @@ -515,7 +515,7 @@ void testFaraoUseCaseManualExport() throws IOException {
.setVersion(exportedVersion)
.setModelingAuthoritySet("myModellingAuthority");
exportParams.put(CgmesExport.PROFILES, List.of("SSH"));
n.write("CGMES", exportParams, new FileDataSource(tmpFolder, basename + "_" + country));
n.write("CGMES", exportParams, new DirectoryDataSource(tmpFolder, basename + "_" + country));
}

// In the main network, CREATE the metadata for SV and export it
Expand All @@ -531,7 +531,7 @@ void testFaraoUseCaseManualExport() throws IOException {
.add()
.add();
exportParams.put(CgmesExport.PROFILES, List.of("SV"));
DataSource dataSource = new FileDataSource(tmpFolder, basename);
DataSource dataSource = new DirectoryDataSource(tmpFolder, basename);
cgmNetwork.write("CGMES", exportParams, dataSource);

int expectedOutputVersion = exportedVersion;
Expand Down
Loading