Datasources

action-ial/action-ial-simulator/src/main/java/com/powsybl/action/ial/simulator/loadflow/CaseExporter.java

@@ -67,7 +67,7 @@ public void loadFlowConverged(RunningContext runningContext, List<LimitViolation
     }
 
     private void exportNetwork(RunningContext context) {
-        DataSource dataSource = DataSourceUtil.createDataSource(outputCaseFolder, getBasename(context.getContingency(), context.getRound()), compressionFormat, null);
+        DataSource dataSource = DataSourceUtil.createBaseNameFilteredDirectoryDataSource(outputCaseFolder, getBasename(context.getContingency(), context.getRound()), compressionFormat);
         context.getNetwork().write(loader, outputCaseFormat, new Properties(), dataSource);
     }
 

ampl-executor/src/main/java/com/powsybl/ampl/executor/AmplModelExecutionHandler.java

@@ -10,7 +10,7 @@
 import com.powsybl.ampl.converter.*;
 import com.powsybl.commons.PowsyblException;
 import com.powsybl.commons.datasource.DataSource;
-import com.powsybl.commons.datasource.FileDataSource;
+import com.powsybl.commons.datasource.DirectoryDataSource;
 import com.powsybl.commons.util.StringToIntMapper;
 import com.powsybl.computation.*;
 import com.powsybl.iidm.network.Network;
@@ -21,11 +21,7 @@
 import java.io.*;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.*;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
+import java.util.*;
 
 /**
  * This executionHandler will run an ampl model on a network.
@@ -97,7 +93,7 @@ private void exportAmplParameters(Path workingDir) throws IOException {
     }
 
     private void exportNetworkAsAmpl(Path workingDir) {
-        DataSource networkExportDataSource = new FileDataSource(workingDir, this.model.getNetworkDataPrefix());
+        DataSource networkExportDataSource = new DirectoryDataSource(workingDir, this.model.getNetworkDataPrefix());
         if (parameters.getAmplExportConfig() != null) {
             new AmplExporter().export(network, parameters.getAmplExportConfig(), networkExportDataSource);
         } else {
@@ -180,7 +176,7 @@ public List<CommandExecution> before(Path workingDir) throws IOException {
     @Override
     public AmplResults after(Path workingDir, ExecutionReport report) throws IOException {
         super.after(workingDir.toAbsolutePath(), report);
-        DataSource networkAmplResults = new FileDataSource(workingDir, this.model.getOutputFilePrefix());
+        DataSource networkAmplResults = new DirectoryDataSource(workingDir, this.model.getOutputFilePrefix());
         AmplNetworkReader reader = new AmplNetworkReader(networkAmplResults, this.network, this.model.getVariant(),
                 mapper, this.model.getNetworkUpdaterFactory(), this.model.getOutputFormat());
         Map<String, String> indicators = readIndicators(reader);

cgmes/cgmes-completion/src/main/java/com/powsybl/cgmes/completion/CreateMissingContainersPreProcessor.java

@@ -22,7 +22,7 @@
 import com.powsybl.cgmes.model.triplestore.CgmesModelTripleStore;
 import com.powsybl.commons.PowsyblException;
 import com.powsybl.commons.config.PlatformConfig;
-import com.powsybl.commons.datasource.ZipFileDataSource;
+import com.powsybl.commons.datasource.ZipArchiveDataSource;
 import com.powsybl.commons.parameters.Parameter;
 import com.powsybl.commons.parameters.ParameterDefaultValueConfig;
 import com.powsybl.commons.parameters.ParameterType;
@@ -132,7 +132,7 @@ private static void prepareAndReadFixesUsingZipFile(CgmesModel cgmes, String bas
         LOG.info("Missing voltage levels: {}", missingVoltageLevels);
         if (!missingVoltageLevels.isEmpty()) {
             buildZipFileWithFixes(cgmes, missingVoltageLevels, fixesFile, basename);
-            cgmes.read(new ZipFileDataSource(fixesFile), ReportNode.NO_OP);
+            cgmes.read(new ZipArchiveDataSource(fixesFile), ReportNode.NO_OP);
         }
         Set<String> missingVoltageLevelsAfterFix = findMissingVoltageLevels(cgmes);
         if (!missingVoltageLevelsAfterFix.isEmpty()) {

cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/CgmesImport.java

@@ -15,10 +15,7 @@
 import com.powsybl.cgmes.model.*;
 import com.powsybl.commons.PowsyblException;
 import com.powsybl.commons.config.PlatformConfig;
-import com.powsybl.commons.datasource.DataSource;
-import com.powsybl.commons.datasource.DataSourceUtil;
-import com.powsybl.commons.datasource.GenericReadOnlyDataSource;
-import com.powsybl.commons.datasource.ReadOnlyDataSource;
+import com.powsybl.commons.datasource.*;
 import com.powsybl.commons.parameters.Parameter;
 import com.powsybl.commons.parameters.ParameterDefaultValueConfig;
 import com.powsybl.commons.parameters.ParameterScope;
@@ -170,7 +167,7 @@ private Network importData1(ReadOnlyDataSource ds, NetworkFactory networkFactory
         return new Conversion(cgmes, config(p), activatedPreProcessors(p), activatedPostProcessors(p), networkFactory).convert(conversionReportNode);
     }
 
-    static class FilteredReadOnlyDataSource implements ReadOnlyDataSource {
+    static class FilteredReadOnlyDataSource extends AbstractReadOnlyDataSource {
         private final ReadOnlyDataSource ds;
         private final Predicate<String> filter;
 
@@ -186,12 +183,17 @@ public String getBaseName() {
 
         @Override
         public boolean exists(String suffix, String ext) throws IOException {
-            return ds.exists(suffix, ext) && filter.test(DataSourceUtil.getFileName(getBaseName(), suffix, ext));
+            return super.exists(DataSourceUtil.getFileName(getBaseName(), suffix, ext));
         }
 
         @Override
-        public boolean exists(String fileName) throws IOException {
-            return ds.exists(fileName) && filter.test(fileName);
+        public boolean existsStrict(String suffix, String ext) throws IOException {
+            return existsStrict(DataSourceUtil.getFileName(getBaseName(), suffix, ext));
+        }
+
+        @Override
+        protected boolean checkFileExistence(String fileName, boolean checkConsistencyWithDataSource) throws IOException {
+            return (checkConsistencyWithDataSource ? ds.existsStrict(fileName) : ds.exists(fileName)) && filter.test(fileName);
         }
 
         @Override

cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/ConversionTester.java

@@ -17,11 +17,14 @@
 import com.powsybl.cgmes.conversion.Conversion;
 import com.powsybl.cgmes.conversion.test.network.compare.Comparison;
 import com.powsybl.cgmes.conversion.test.network.compare.ComparisonConfig;
-import com.powsybl.cgmes.model.*;
+import com.powsybl.cgmes.model.CgmesModel;
+import com.powsybl.cgmes.model.CgmesOnDataSource;
+import com.powsybl.cgmes.model.CgmesSubset;
+import com.powsybl.cgmes.model.GridModelReference;
 import com.powsybl.commons.datasource.DataSource;
-import com.powsybl.commons.datasource.FileDataSource;
+import com.powsybl.commons.datasource.DirectoryDataSource;
 import com.powsybl.commons.datasource.ReadOnlyDataSource;
-import com.powsybl.commons.datasource.ZipFileDataSource;
+import com.powsybl.commons.datasource.ZipArchiveDataSource;
 import com.powsybl.iidm.network.Network;
 import com.powsybl.iidm.network.impl.NetworkFactoryImpl;
 import com.powsybl.iidm.serde.XMLExporter;
@@ -165,17 +168,17 @@ private static void exportXiidm(String name, String impl, Network expected, Netw
         XMLExporter xmlExporter = new XMLExporter();
         // Last component of the path is the name for the exported XML
         if (expected != null) {
-            xmlExporter.export(expected, null, new FileDataSource(path, "expected"));
+            xmlExporter.export(expected, null, new DirectoryDataSource(path, "expected"));
         }
         if (actual != null) {
-            xmlExporter.export(actual, null, new FileDataSource(path, "actual"));
+            xmlExporter.export(actual, null, new DirectoryDataSource(path, "actual"));
         }
     }
 
     private static void exportCgmes(String name, String impl, Network network) throws IOException {
         String name1 = name.replace('/', '-');
         Path path = Files.createTempDirectory("temp-export-cgmes-" + name1 + "-" + impl + "-");
-        new CgmesExport().export(network, null, new FileDataSource(path, "foo"));
+        new CgmesExport().export(network, null, new DirectoryDataSource(path, "foo"));
     }
 
     private static String subsetFromName(String name) {
@@ -197,7 +200,7 @@ private void testExportImportCgmes(Network network, ReadOnlyDataSource originalD
         Path path = fs.getPath("temp-export-cgmes");
         Files.createDirectories(path);
         String baseName = originalDs.getBaseName();
-        DataSource ds = new ZipFileDataSource(path, baseName);
+        DataSource ds = new ZipArchiveDataSource(path, baseName);
 
         // Copy the original files to the temporary destination, ensuring a normalized name
         for (String name : new CgmesOnDataSource(originalDs).names()) {

cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/ImportExportPerformanceTest.java

@@ -17,7 +17,7 @@
 import com.powsybl.cgmes.model.GridModelReference;
 import com.powsybl.cgmes.model.test.Cim14SmallCasesCatalog;
 import com.powsybl.commons.datasource.DataSource;
-import com.powsybl.commons.datasource.FileDataSource;
+import com.powsybl.commons.datasource.DirectoryDataSource;
 import com.powsybl.commons.datasource.ReadOnlyDataSource;
 import com.powsybl.iidm.network.Network;
 import com.powsybl.iidm.network.NetworkFactory;
@@ -81,7 +81,7 @@ private void importExport(String ts, ReadOnlyDataSource ds, FileSystem fs) throw
         CgmesExport e = new CgmesExport();
         Path exportFolder = fs.getPath("impl-" + ts);
         Files.createDirectories(exportFolder);
-        DataSource exportDataSource = new FileDataSource(exportFolder, "");
+        DataSource exportDataSource = new DirectoryDataSource(exportFolder, "");
         Properties exportParameters = new Properties();
         exportParameters.put(CgmesExport.CIM_VERSION, "16");
         e.export(n, exportParameters, exportDataSource);

cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/export/CgmesExportTest.java

@@ -34,7 +34,10 @@
 import javax.xml.stream.XMLStreamReader;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.file.*;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.util.List;
 import java.util.Properties;
 
@@ -477,9 +480,9 @@ void testModelDescription() throws IOException {
 
         try (FileSystem fileSystem = Jimfs.newFileSystem(Configuration.unix())) {
             Path tmpDir = Files.createDirectory(fileSystem.getPath("tmp"));
-            ZipFileDataSource zip = new ZipFileDataSource(tmpDir.resolve("."), "output");
+            ZipArchiveDataSource zip = new ZipArchiveDataSource(tmpDir.resolve("."), "output");
             new CgmesExport().export(network, params, zip);
-            Network network2 = Network.read(new GenericReadOnlyDataSource(tmpDir.resolve("output.zip")), importParams);
+            Network network2 = Network.read(new GenericReadOnlyDataSource(tmpDir, "output"), importParams);
             CgmesMetadataModel sshMetadata = network2
                     .getExtension(CgmesMetadataModels.class)
                     .getModelForSubset(CgmesSubset.STATE_VARIABLES)
@@ -497,9 +500,9 @@ void testModelVersion() throws IOException {
 
         try (FileSystem fileSystem = Jimfs.newFileSystem(Configuration.unix())) {
             Path tmpDir = Files.createDirectory(fileSystem.getPath("tmp"));
-            ZipFileDataSource zip = new ZipFileDataSource(tmpDir.resolve("."), "output");
+            ZipArchiveDataSource zip = new ZipArchiveDataSource(tmpDir.resolve("."), "output");
             new CgmesExport().export(network, params, zip);
-            Network network2 = Network.read(new GenericReadOnlyDataSource(tmpDir.resolve("output.zip")), importParams);
+            Network network2 = Network.read(new GenericReadOnlyDataSource(tmpDir, "output"), importParams);
             CgmesMetadataModel sshMetadata = network2.getExtension(CgmesMetadataModels.class).getModelForSubset(CgmesSubset.STEADY_STATE_HYPOTHESIS).orElseThrow();
             assertEquals(9, sshMetadata.getVersion());
             CgmesMetadataModel svMetadata = network2.getExtension(CgmesMetadataModels.class).getModelForSubset(CgmesSubset.STATE_VARIABLES).orElseThrow();
@@ -519,11 +522,11 @@ void testModelDescriptionClosingXML() throws IOException {
 
         try (FileSystem fileSystem = Jimfs.newFileSystem(Configuration.unix())) {
             Path tmpDir = Files.createDirectory(fileSystem.getPath("tmp"));
-            ZipFileDataSource zip = new ZipFileDataSource(tmpDir.resolve("."), "output");
+            ZipArchiveDataSource zip = new ZipArchiveDataSource(tmpDir.resolve("."), "output");
             new CgmesExport().export(network, params, zip);
 
             // check network can be reimported and that ModelDescription still includes end-tag
-            Network network2 = Network.read(new GenericReadOnlyDataSource(tmpDir.resolve("output.zip")), importParams);
+            Network network2 = Network.read(new GenericReadOnlyDataSource(tmpDir, "output"), importParams);
 
             CgmesMetadataModel sshMetadata = network2.getExtension(CgmesMetadataModels.class).getModelForSubset(CgmesSubset.STEADY_STATE_HYPOTHESIS).orElseThrow();
             assertEquals(modelDescription, sshMetadata.getDescription());
@@ -561,7 +564,7 @@ void testExportWithModelingAuthorityFromReferenceData() throws IOException {
             Properties exportParams = new Properties();
             // It is enough to check that the MAS has been set correctly in the EQ instance file
             exportParams.put(CgmesExport.PROFILES, "EQ");
-            new CgmesExport(platformConfig).export(network, exportParams, new FileDataSource(tmpDir, network.getNameOrId()));
+            new CgmesExport(platformConfig).export(network, exportParams, new DirectoryDataSource(tmpDir, network.getNameOrId()));
 
             String eq = Files.readString(tmpDir.resolve(network.getNameOrId() + "_EQ.xml"));
             assertTrue(eq.contains("modelingAuthoritySet>http://www.elia.be/OperationalPlanning"));
@@ -625,7 +628,7 @@ void testCanGeneratorControl() throws IOException {
             Properties exportParams = new Properties();
             exportParams.put(CgmesExport.PROFILES, "EQ");
             // network.write("CGMES", null, tmpDir.resolve(baseName));
-            new CgmesExport().export(network, exportParams, new FileDataSource(tmpDir, baseName));
+            new CgmesExport().export(network, exportParams, new DirectoryDataSource(tmpDir, baseName));
             String eq = Files.readString(tmpDir.resolve(baseName + "_EQ.xml"));
 
             // Check that RC are exported properly
@@ -644,7 +647,7 @@ void testCanGeneratorControl() throws IOException {
             generatorNoRcc.setTargetV(Double.NaN);
 
             //network.write("CGMES", null, tmpDir.resolve(baseName));
-            new CgmesExport().export(network, exportParams, new FileDataSource(tmpDir, baseName));
+            new CgmesExport().export(network, exportParams, new DirectoryDataSource(tmpDir, baseName));
             eq = Files.readString(tmpDir.resolve(baseName + "_EQ.xml"));
             assertFalse(eq.contains("3a3b27be-b18b-4385-b557-6735d733baf0_RC"));
             assertFalse(eq.contains("550ebe0d-f2b2-48c1-991f-cebea43a21aa_RC"));
@@ -665,7 +668,7 @@ void testCanGeneratorControl() throws IOException {
             mmrlAdder.setMaxQ(mmrl.getMinQ());
             mmrlAdder.add();
 
-            new CgmesExport().export(network, exportParams, new FileDataSource(tmpDir, baseName));
+            new CgmesExport().export(network, exportParams, new DirectoryDataSource(tmpDir, baseName));
             eq = Files.readString(tmpDir.resolve(baseName + "_EQ.xml"));
             assertFalse(eq.contains("3a3b27be-b18b-4385-b557-6735d733baf0_RC"));
             assertFalse(eq.contains("550ebe0d-f2b2-48c1-991f-cebea43a21aa_RC"));

cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/export/CgmesNamingStrategyTest.java

@@ -9,14 +9,19 @@
 
 import com.powsybl.cgmes.conformity.CgmesConformity1Catalog;
 import com.powsybl.cgmes.conformity.CgmesConformity1ModifiedCatalog;
-import com.powsybl.cgmes.conversion.*;
+import com.powsybl.cgmes.conversion.CgmesExport;
+import com.powsybl.cgmes.conversion.CgmesModelExtension;
+import com.powsybl.cgmes.conversion.Conversion;
 import com.powsybl.cgmes.conversion.export.CgmesExportUtil;
 import com.powsybl.cgmes.conversion.naming.NamingStrategyFactory;
 import com.powsybl.cgmes.model.CgmesModel;
 import com.powsybl.cgmes.model.CgmesNames;
 import com.powsybl.commons.datasource.*;
 import com.powsybl.commons.test.AbstractSerDeTest;
-import com.powsybl.iidm.network.*;
+import com.powsybl.iidm.network.Identifiable;
+import com.powsybl.iidm.network.IdentifiableType;
+import com.powsybl.iidm.network.Network;
+import com.powsybl.iidm.network.NetworkFactory;
 import com.powsybl.iidm.network.extensions.SlackTerminal;
 import com.powsybl.iidm.serde.NetworkSerDe;
 import com.powsybl.iidm.serde.XMLImporter;
@@ -29,12 +34,16 @@
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.*;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.Properties;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
  * @author Marcos de Miguel {@literal <demiguelm at aia.es>}
@@ -223,7 +232,7 @@ private DataSource tmpDataSource(String folder, String baseName) throws IOExcept
             FileUtils.cleanDirectory(exportFolder.toFile());
         }
         Files.createDirectories(exportFolder);
-        return new FileDataSource(exportFolder, baseName);
+        return new DirectoryDataSource(exportFolder, baseName);
     }
 
     private void copyBoundary(String outputFolderName, String baseName, ReadOnlyDataSource originalDataSource) throws IOException {

cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/export/CommonGridModelExportTest.java

@@ -15,11 +15,12 @@
 import com.powsybl.cgmes.model.CgmesSubset;
 import com.powsybl.commons.PowsyblException;
 import com.powsybl.commons.datasource.DataSource;
-import com.powsybl.commons.datasource.FileDataSource;
+import com.powsybl.commons.datasource.DirectoryDataSource;
 import com.powsybl.commons.datasource.MemDataSource;
 import com.powsybl.commons.datasource.ReadOnlyDataSource;
 import com.powsybl.commons.test.AbstractSerDeTest;
-import com.powsybl.iidm.network.*;
+import com.powsybl.iidm.network.Country;
+import com.powsybl.iidm.network.Network;
 import org.junit.jupiter.api.Test;
 
 import javax.xml.stream.XMLInputFactory;
@@ -475,7 +476,7 @@ void testFaraoUseCaseManualExport() throws IOException {
                     .setVersion(exportedVersion)
                     .setModelingAuthoritySet("myModellingAuthority");
             exportParams.put(CgmesExport.PROFILES, List.of("SSH"));
-            n.write("CGMES", exportParams, new FileDataSource(tmpFolder, basename + "_" + country));
+            n.write("CGMES", exportParams, new DirectoryDataSource(tmpFolder, basename + "_" + country));
         }
 
         // In the main network, CREATE the metadata for SV and export it
@@ -491,7 +492,7 @@ void testFaraoUseCaseManualExport() throws IOException {
                 .add()
                 .add();
         exportParams.put(CgmesExport.PROFILES, List.of("SV"));
-        DataSource dataSource = new FileDataSource(tmpFolder, basename);
+        DataSource dataSource = new DirectoryDataSource(tmpFolder, basename);
         cgmNetwork.write("CGMES", exportParams, dataSource);
 
         int expectedOutputVersion = exportedVersion;