From eda4baf15d5702b9208b01138b053da75196a1f1 Mon Sep 17 00:00:00 2001 From: HARPER Jon Date: Wed, 26 Jun 2024 17:32:36 +0200 Subject: [PATCH] Disambiguate files on import Signed-off-by: HARPER Jon --- .../cgmes/model/CgmesOnDataSource.java | 48 +++++++++++++++++-- .../commons/datasource/DataSourceUtil.java | 12 ++++- .../commons/datasource/FileDataSource.java | 18 ++++++- .../datasource/ReadOnlyDataSource.java | 4 ++ .../ieeecdf/converter/IeeeCdfImporter.java | 6 +++ .../iidm/serde/AbstractTreeDataImporter.java | 10 ++++ .../matpower/converter/MatpowerImporter.java | 6 +++ .../converter/PowerFactoryImporter.java | 14 +++++- .../powsybl/psse/converter/PsseImporter.java | 9 ++++ .../powsybl/ucte/converter/UcteImporter.java | 9 ++++ 10 files changed, 129 insertions(+), 7 deletions(-) diff --git a/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/CgmesOnDataSource.java b/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/CgmesOnDataSource.java index 15ac210058a..28af7ee1e1b 100644 --- a/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/CgmesOnDataSource.java +++ b/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/CgmesOnDataSource.java @@ -24,6 +24,9 @@ * @author Luma Zamarreño {@literal } */ public class CgmesOnDataSource { + + private static final String EXTENSION = "xml"; + public CgmesOnDataSource(ReadOnlyDataSource ds) { this.dataSource = ds; } @@ -33,25 +36,62 @@ public ReadOnlyDataSource dataSource() { } public boolean exists() { + // if given a main file with our extension, check that the mainfile is our format + if (EXTENSION.equals(dataSource.getBaseExtension())) { + try (InputStream is = dataSource.newInputStream(null, EXTENSION)) { + if (!existsNamespaces(NamespaceReader.namespaces(is))) { + return false; + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + // if there is an extension and it's not compatible, don't import + } else if (!dataSource.getBaseExtension().isEmpty()) { + return false; + } + // check that RDF and CIM16 are defined as namespaces in the data source Set foundNamespaces = namespaces(); - if (!foundNamespaces.contains(RDF_NAMESPACE)) { + return existsNamespaces(foundNamespaces); + } + + private boolean existsNamespaces(Set namespaces) { + if (!namespaces.contains(RDF_NAMESPACE)) { return false; } // FIXME(Luma) This is legacy behaviour, we do not consider CIM14 valid in this check // But I think we do not need to support 14 separately? - return foundNamespaces.contains(CIM_16_NAMESPACE) || foundNamespaces.contains(CIM_100_NAMESPACE); + return namespaces.contains(CIM_16_NAMESPACE) || namespaces.contains(CIM_100_NAMESPACE); } public boolean existsCim14() { + // if given a main file with our extension, check that the mainfile is our format + if (EXTENSION.equals(dataSource.getBaseExtension())) { + try (InputStream is = dataSource.newInputStream(null, EXTENSION)) { + if (!existsNamespacesCim14(NamespaceReader.namespaces(is))) { + return false; + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + // if given a main file with an extension and it's not compatible, don't import + // to avoid importing a file when the user specified another + } else if (!dataSource.getBaseExtension().isEmpty()) { + return false; + } + // check that RDF and CIM16 are defined as namespaces in the data source Set foundNamespaces = namespaces(); - if (!foundNamespaces.contains(RDF_NAMESPACE)) { + return existsNamespacesCim14(foundNamespaces); + } + + private boolean existsNamespacesCim14(Set namespaces) { + if (!namespaces.contains(RDF_NAMESPACE)) { return false; } // FIXME(Luma) This is legacy behaviour, we do not consider CIM14 valid in this check // But I think we do not need to support 14 separately? - if (!foundNamespaces.contains(CIM_14_NAMESPACE)) { + if (!namespaces.contains(CIM_14_NAMESPACE)) { return false; } diff --git a/commons/src/main/java/com/powsybl/commons/datasource/DataSourceUtil.java b/commons/src/main/java/com/powsybl/commons/datasource/DataSourceUtil.java index b49896c896a..f8b9c66952d 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/DataSourceUtil.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/DataSourceUtil.java @@ -39,6 +39,16 @@ static String getBaseName(String fileName) { return pos == -1 ? fileName : fileName.substring(0, pos); } + static String getMainExtension(String fileName) { + Objects.requireNonNull(fileName); + int firstpos = fileName.indexOf('.'); // find first dot in case of double extension (.xml.gz) + int secondpos = fileName.indexOf('.', firstpos + 1); // find second dot in case of double extension (.xml.gz) + if (secondpos == -1) { + secondpos = fileName.length(); + } + return firstpos == -1 ? "" : fileName.substring(firstpos + 1, secondpos); + } + static DataSource createDataSource(Path directory, String basename, CompressionFormat compressionExtension, DataSourceObserver observer) { Objects.requireNonNull(directory); Objects.requireNonNull(basename); @@ -78,7 +88,7 @@ static DataSource createDataSource(Path directory, String fileNameOrBaseName, Da } else if (fileNameOrBaseName.endsWith(".bz2")) { return new Bzip2FileDataSource(directory, getBaseName(fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 4)), observer); } else { - return new FileDataSource(directory, getBaseName(fileNameOrBaseName), observer); + return new FileDataSource(directory, getBaseName(fileNameOrBaseName), getMainExtension(fileNameOrBaseName), observer); } } } diff --git a/commons/src/main/java/com/powsybl/commons/datasource/FileDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/FileDataSource.java index 2e30202a2f3..cf22669f4e8 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/FileDataSource.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/FileDataSource.java @@ -29,15 +29,26 @@ public class FileDataSource implements DataSource { private final String baseName; + private final String baseExtension; + private final DataSourceObserver observer; public FileDataSource(Path directory, String baseName) { - this(directory, baseName, null); + this(directory, baseName, "", null); } public FileDataSource(Path directory, String baseName, DataSourceObserver observer) { + this(directory, baseName, "", observer); + } + + public FileDataSource(Path directory, String baseName, String baseExtension) { + this(directory, baseName, baseExtension, null); + } + + public FileDataSource(Path directory, String baseName, String baseExtension, DataSourceObserver observer) { this.directory = Objects.requireNonNull(directory); this.baseName = Objects.requireNonNull(baseName); + this.baseExtension = Objects.requireNonNull(baseExtension); this.observer = observer; } @@ -46,6 +57,11 @@ public String getBaseName() { return baseName; } + @Override + public String getBaseExtension() { + return baseExtension; + } + protected String getCompressionExt() { return COMPRESSION_EXT; } diff --git a/commons/src/main/java/com/powsybl/commons/datasource/ReadOnlyDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/ReadOnlyDataSource.java index 2e21b00c0b8..260e636c8ac 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/ReadOnlyDataSource.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/ReadOnlyDataSource.java @@ -18,6 +18,10 @@ public interface ReadOnlyDataSource { String getBaseName(); + default String getBaseExtension() { + return ""; + } + boolean exists(String suffix, String ext) throws IOException; boolean exists(String fileName) throws IOException; diff --git a/ieee-cdf/ieee-cdf-converter/src/main/java/com/powsybl/ieeecdf/converter/IeeeCdfImporter.java b/ieee-cdf/ieee-cdf-converter/src/main/java/com/powsybl/ieeecdf/converter/IeeeCdfImporter.java index 0d410b6082e..b885467ef88 100644 --- a/ieee-cdf/ieee-cdf-converter/src/main/java/com/powsybl/ieeecdf/converter/IeeeCdfImporter.java +++ b/ieee-cdf/ieee-cdf-converter/src/main/java/com/powsybl/ieeecdf/converter/IeeeCdfImporter.java @@ -79,6 +79,12 @@ public String getComment() { @Override public boolean exists(ReadOnlyDataSource dataSource) { + // if given a main file with an extension and it's not compatible, don't import + // to avoid importing a file when the user specified another + if (!dataSource.getBaseExtension().isEmpty()) { + return false; + } + try { if (dataSource.exists(null, EXT)) { try (BufferedReader reader = new BufferedReader(new InputStreamReader(dataSource.newInputStream(null, EXT)))) { diff --git a/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/AbstractTreeDataImporter.java b/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/AbstractTreeDataImporter.java index 621c93a5a43..a8c4c8e736e 100644 --- a/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/AbstractTreeDataImporter.java +++ b/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/AbstractTreeDataImporter.java @@ -31,6 +31,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.UncheckedIOException; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -96,6 +97,15 @@ public List getParameters() { } private String findExtension(ReadOnlyDataSource dataSource) throws IOException { + // if given a main file compatible with our extensions, try that as the mainfile. + if (Arrays.asList(getExtensions()).contains(dataSource.getBaseExtension())) { + return dataSource.getBaseExtension(); + // if given a main file with an extension and it's not compatible, don't import + // to avoid importing a file when the user specified another + } else if (!dataSource.getBaseExtension().isEmpty()) { + return null; + } + for (String ext : getExtensions()) { if (dataSource.exists(null, ext)) { return ext; diff --git a/matpower/matpower-converter/src/main/java/com/powsybl/matpower/converter/MatpowerImporter.java b/matpower/matpower-converter/src/main/java/com/powsybl/matpower/converter/MatpowerImporter.java index 40ca6a38e3a..9be5d9d88f2 100644 --- a/matpower/matpower-converter/src/main/java/com/powsybl/matpower/converter/MatpowerImporter.java +++ b/matpower/matpower-converter/src/main/java/com/powsybl/matpower/converter/MatpowerImporter.java @@ -515,6 +515,12 @@ private static boolean reactiveLimitsAreOk(double minQ, double maxQ) { @Override public boolean exists(ReadOnlyDataSource dataSource) { + // if given a main file with an extension and it's not compatible, don't import + // to avoid importing a file when the user specified another + if (!dataSource.getBaseExtension().isEmpty()) { + return false; + } + try { return dataSource.exists(null, MatpowerConstants.EXT); } catch (IOException e) { diff --git a/powerfactory/powerfactory-converter/src/main/java/com/powsybl/powerfactory/converter/PowerFactoryImporter.java b/powerfactory/powerfactory-converter/src/main/java/com/powsybl/powerfactory/converter/PowerFactoryImporter.java index c3aaf1b2bf7..602759a6a08 100644 --- a/powerfactory/powerfactory-converter/src/main/java/com/powsybl/powerfactory/converter/PowerFactoryImporter.java +++ b/powerfactory/powerfactory-converter/src/main/java/com/powsybl/powerfactory/converter/PowerFactoryImporter.java @@ -62,7 +62,19 @@ public String getComment() { } private Optional> findProjectLoader(ReadOnlyDataSource dataSource) { - for (PowerFactoryDataLoader studyCaseLoader : PowerFactoryDataLoader.find(StudyCase.class)) { + List> loaders = PowerFactoryDataLoader.find(StudyCase.class); + Map>> extensions = loaders.stream().collect( + Collectors.groupingBy(PowerFactoryDataLoader::getExtension)); + // if given a main file compatible with our extensions, try that as the mainfile. + if (extensions.containsKey(dataSource.getBaseExtension())) { + return Optional.of(extensions.get(dataSource.getBaseExtension()).get(0)); + // if given a main file with an extension and it's not compatible, don't import + // to avoid importing a file when the user specified another + } else if (!dataSource.getBaseExtension().isEmpty()) { + return Optional.empty(); + } + + for (PowerFactoryDataLoader studyCaseLoader : loaders) { try { if (dataSource.exists(null, studyCaseLoader.getExtension())) { return Optional.of(studyCaseLoader); diff --git a/psse/psse-converter/src/main/java/com/powsybl/psse/converter/PsseImporter.java b/psse/psse-converter/src/main/java/com/powsybl/psse/converter/PsseImporter.java index ef1fecaeec0..06ce0fe7972 100644 --- a/psse/psse-converter/src/main/java/com/powsybl/psse/converter/PsseImporter.java +++ b/psse/psse-converter/src/main/java/com/powsybl/psse/converter/PsseImporter.java @@ -82,6 +82,15 @@ public boolean exists(ReadOnlyDataSource dataSource) { } private String findExtension(ReadOnlyDataSource dataSource) throws IOException { + // if given a main file compatible with our extensions, try that as the mainfile. + if (Arrays.asList(EXTENSIONS).contains(dataSource.getBaseExtension())) { + return dataSource.getBaseExtension(); + // if given a main file with an extension and it's not compatible, don't import + // to avoid importing a file when the user specified another + } else if (!dataSource.getBaseExtension().isEmpty()) { + return null; + } + for (String ext : EXTENSIONS) { if (dataSource.exists(null, ext)) { return ext; diff --git a/ucte/ucte-converter/src/main/java/com/powsybl/ucte/converter/UcteImporter.java b/ucte/ucte-converter/src/main/java/com/powsybl/ucte/converter/UcteImporter.java index 2f9db7339ca..c12bb0dfda9 100644 --- a/ucte/ucte-converter/src/main/java/com/powsybl/ucte/converter/UcteImporter.java +++ b/ucte/ucte-converter/src/main/java/com/powsybl/ucte/converter/UcteImporter.java @@ -950,6 +950,15 @@ public List getParameters() { } private String findExtension(ReadOnlyDataSource dataSource, boolean throwException) throws IOException { + // if given a main file compatible with our extensions, try that as the mainfile. + if (Arrays.asList(EXTENSIONS).contains(dataSource.getBaseExtension())) { + return dataSource.getBaseExtension(); + // if given a main file with an extension and it's not compatible, don't import + // to avoid importing a file when the user specified another + } else if (!dataSource.getBaseExtension().isEmpty()) { + return null; + } + for (String ext : EXTENSIONS) { if (dataSource.exists(null, ext)) { return ext;