diff --git a/commons/src/main/java/com/powsybl/commons/datasource/AbstractArchiveDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/AbstractArchiveDataSource.java index 6871f2b1149..4e2bc0bcbe0 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/AbstractArchiveDataSource.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/AbstractArchiveDataSource.java @@ -8,6 +8,7 @@ package com.powsybl.commons.datasource; import java.nio.file.Path; +import java.util.Objects; /** * @author Nicolas Rol {@literal } @@ -26,4 +27,17 @@ public abstract class AbstractArchiveDataSource extends AbstractFileSystemDataSo protected Path getArchiveFilePath() { return directory.resolve(archiveFileName); } + + protected ArchiveFormat getArchiveFormat() { + return archiveFormat; + } + + protected abstract boolean entryExists(Path archiveFilePath, String fileName); + + @Override + public boolean exists(String fileName) { + Objects.requireNonNull(fileName); + Path archiveFilePath = getArchiveFilePath(); + return entryExists(archiveFilePath, fileName); + } } diff --git a/commons/src/main/java/com/powsybl/commons/datasource/ArchiveFormat.java b/commons/src/main/java/com/powsybl/commons/datasource/ArchiveFormat.java index 8de2889c47a..c9bfe975c93 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/ArchiveFormat.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/ArchiveFormat.java @@ -15,7 +15,8 @@ * @author Nicolas Rol {@literal } */ public enum ArchiveFormat { - ZIP("zip"); + ZIP("zip"), + TAR("tar"); ArchiveFormat(String extension) { this.extension = Objects.requireNonNull(extension); diff --git a/commons/src/main/java/com/powsybl/commons/datasource/DataSourceBuilder.java b/commons/src/main/java/com/powsybl/commons/datasource/DataSourceBuilder.java index 9736c8d3cbe..c3f4746e145 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/DataSourceBuilder.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/DataSourceBuilder.java @@ -69,6 +69,10 @@ DataSource build() { // Create the datasource if (compressionFormat == CompressionFormat.ZIP || archiveFormat == ArchiveFormat.ZIP) { return buildZip(); + } else if (archiveFormat == ArchiveFormat.TAR) { + return archiveFileName == null ? + new TarArchiveDataSource(directory, baseName, dataExtension, compressionFormat, observer) : + new TarArchiveDataSource(directory, archiveFileName, baseName, dataExtension, compressionFormat, observer); } else if (compressionFormat == null) { return new DirectoryDataSource(directory, baseName, dataExtension, observer); } else { diff --git a/commons/src/main/java/com/powsybl/commons/datasource/FileInformation.java b/commons/src/main/java/com/powsybl/commons/datasource/FileInformation.java index 2cb17ece723..15bad57ac66 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/FileInformation.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/FileInformation.java @@ -56,10 +56,21 @@ private void computeInformation(String fileName, boolean dataSourceInitializatio // File name without the compression extension String fileNameWithoutCompressionExtension = compressionFormat == null ? fileName : fileName.substring(0, currentDotIndex); + // Last dot index + currentDotIndex = fileNameWithoutCompressionExtension.lastIndexOf('.'); + // Archive extension String fileNameWithoutCompressionNorArchive; - archiveFormat = compressionFormat == CompressionFormat.ZIP ? ArchiveFormat.ZIP : null; - fileNameWithoutCompressionNorArchive = fileNameWithoutCompressionExtension; + if (compressionFormat == CompressionFormat.ZIP) { + archiveFormat = ArchiveFormat.ZIP; + fileNameWithoutCompressionNorArchive = fileNameWithoutCompressionExtension; + } else if (ArchiveFormat.TAR.getExtension().equals(fileNameWithoutCompressionExtension.substring(currentDotIndex + 1))) { + archiveFormat = ArchiveFormat.TAR; + fileNameWithoutCompressionNorArchive = fileNameWithoutCompressionExtension.substring(0, currentDotIndex); + } else { + archiveFormat = null; + fileNameWithoutCompressionNorArchive = fileNameWithoutCompressionExtension; + } // Last dot index currentDotIndex = fileNameWithoutCompressionNorArchive.lastIndexOf('.'); diff --git a/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java new file mode 100644 index 00000000000..c79f7872f72 --- /dev/null +++ b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java @@ -0,0 +1,330 @@ +/* + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * SPDX-License-Identifier: MPL-2.0 + */ +package com.powsybl.commons.datasource; + +import com.google.common.io.ByteStreams; +import com.powsybl.commons.io.ForwardingInputStream; +import com.powsybl.commons.io.ForwardingOutputStream; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; +import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream; +import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * @author Nicolas Rol {@literal } + */ +public class TarArchiveDataSource extends AbstractArchiveDataSource { + + public TarArchiveDataSource(Path directory, String tarFileName, String baseName, String dataExtension, CompressionFormat compressionFormat, DataSourceObserver observer) { + super(directory, tarFileName, baseName, dataExtension, compressionFormat, ArchiveFormat.TAR, observer); + } + + public TarArchiveDataSource(Path directory, String tarFileName, String baseName, String dataExtension, CompressionFormat compressionFormat) { + this(directory, tarFileName, baseName, dataExtension, compressionFormat, null); + } + + public TarArchiveDataSource(Path directory, String baseName, String dataExtension, CompressionFormat compressionFormat, DataSourceObserver observer) { + this(directory, + baseName + (dataExtension == null || dataExtension.isEmpty() ? "" : "." + dataExtension) + ".tar" + (compressionFormat == null ? "" : "." + compressionFormat.getExtension()), + baseName, dataExtension, compressionFormat, observer); + } + + public TarArchiveDataSource(Path directory, String baseName, String dataExtension, CompressionFormat compressionFormat) { + this(directory, + baseName + (dataExtension == null || dataExtension.isEmpty() ? "" : "." + dataExtension) + ".tar" + (compressionFormat == null ? "" : "." + compressionFormat.getExtension()), + baseName, dataExtension, compressionFormat, null); + } + + public TarArchiveDataSource(Path directory, String baseName, CompressionFormat compressionFormat, DataSourceObserver observer) { + this(directory, + baseName + ".tar" + (compressionFormat == null ? "" : "." + compressionFormat.getExtension()), + baseName, null, compressionFormat, observer); + } + + public TarArchiveDataSource(Path directory, String baseName, CompressionFormat compressionFormat) { + this(directory, + baseName + ".tar" + (compressionFormat == null ? "" : "." + compressionFormat.getExtension()), + baseName, null, compressionFormat, null); + } + + public TarArchiveDataSource(Path directory, String baseName) { + this(directory, + baseName + ".tar", + baseName, null, null, null); + } + + public TarArchiveDataSource(Path tarFile) { + this(tarFile.getParent(), new FileInformation(tarFile.getFileName().toString(), false)); + } + + private TarArchiveDataSource(Path directory, FileInformation fileInformation) { + this(directory, fileInformation.getBaseName(), fileInformation.getDataExtension(), fileInformation.getCompressionFormat()); + } + + /** + * {@inheritDoc} + * + *

Files are here located in the archive.

+ */ + @Override + public Set listNames(String regex) throws IOException { + // Initialize variables + Pattern p = Pattern.compile(regex); + Set names = new HashSet<>(); + Path tarFilePath = getArchiveFilePath(); + + // Explore the archive + try (BufferedInputStream inputStream = new BufferedInputStream(Files.newInputStream(tarFilePath)); + InputStream cis = getCompressedInputStream(inputStream, compressionFormat); + TarArchiveInputStream tar = new TarArchiveInputStream(cis)) { + ArchiveEntry entry; + while ((entry = tar.getNextEntry()) != null) { + if (!entry.isDirectory() + && p.matcher(entry.getName()).matches()) { + names.add(entry.getName()); + } + } + } + return names; + } + + protected boolean entryExists(Path tarFilePath, String fileName) { + if (Files.exists(tarFilePath)) { + try (InputStream fis = Files.newInputStream(tarFilePath); + BufferedInputStream bis = new BufferedInputStream(fis); + InputStream is = getCompressedInputStream(bis, compressionFormat); + TarArchiveInputStream tais = new TarArchiveInputStream(is)) { + + TarArchiveEntry entry; + while ((entry = tais.getNextEntry()) != null) { + if (entry.getName().equals(fileName)) { + return true; + } + } + return false; + } catch (IOException | UnsupportedOperationException e) { + return false; + } + } + return false; + } + + @Override + public OutputStream newOutputStream(String suffix, String ext, boolean append) throws IOException { + return newOutputStream(DataSourceUtil.getFileName(baseName, suffix, ext), append); + } + + @Override + public OutputStream newOutputStream(String fileName, boolean append) throws IOException { + Objects.requireNonNull(fileName); + if (append) { + throw new UnsupportedOperationException("append not supported in tar file data source"); + } + Path tarFilePath = getArchiveFilePath(); + OutputStream os = new TarEntryOutputStream(tarFilePath, fileName, compressionFormat); + return observer != null ? new ObservableOutputStream(os, tarFilePath + ":" + fileName, observer) : os; + } + + @Override + public InputStream newInputStream(String suffix, String ext) throws IOException { + return newInputStream(DataSourceUtil.getFileName(baseName, suffix, ext)); + } + + @Override + public InputStream newInputStream(String fileName) throws IOException { + Objects.requireNonNull(fileName); + Path tarFilePath = getArchiveFilePath(); + + // If the file is in the archive, we can open it + if (entryExists(tarFilePath, fileName)) { + InputStream is = new TarEntryInputStream(tarFilePath, fileName, compressionFormat); + return observer != null ? new ObservableInputStream(is, tarFilePath + ":" + fileName, observer) : is; + } + return null; + } + + private static final class TarEntryInputStream extends ForwardingInputStream { + + private TarEntryInputStream(Path tarFilePath, String fileName, CompressionFormat compressionFormat) throws IOException { + super(setStreamToFile(getTmpStream(tarFilePath, compressionFormat), fileName)); + } + + private static TarArchiveInputStream getTmpStream(Path tarFilePath, CompressionFormat compressionFormat) throws IOException { + return new TarArchiveInputStream(getCompressedInputStream(new BufferedInputStream(Files.newInputStream(tarFilePath)), compressionFormat)); + } + + private static InputStream setStreamToFile(TarArchiveInputStream tais, String fileName) throws IOException { + TarArchiveEntry entry; + while ((entry = tais.getNextEntry()) != null) { + if (entry.getName().equals(fileName)) { + return tais; + } + } + return null; + } + } + + private static final class TarEntryOutputStream extends ForwardingOutputStream { + + private final Path tarFilePath; + private final String fileName; + private final CompressionFormat compressionFormat; + private boolean closed; + + private TarEntryOutputStream(Path tarFilePath, String fileName, CompressionFormat compressionFormat) throws IOException { + super(getTmpStream(getTmpStreamFilePath(tarFilePath))); + this.tarFilePath = tarFilePath; + this.fileName = fileName; + this.compressionFormat = compressionFormat; + this.closed = false; + } + + private static OutputStream getTmpStream(Path tarFilePath) throws IOException { + return new BufferedOutputStream(Files.newOutputStream(tarFilePath)); + } + + private static Path getTmpStreamFilePath(Path tarFilePath) { + return tarFilePath.getParent().resolve("tmp_stream_" + tarFilePath.getFileName() + ".stream"); + } + + private static TarArchiveOutputStream getTarStream(Path tmpTarFilePath) throws IOException { + return new TarArchiveOutputStream(new BufferedOutputStream(Files.newOutputStream(tmpTarFilePath))); + } + + private static Path getTmpTarFilePath(Path tarFilePath) { + return tarFilePath.getParent().resolve("tmp_" + tarFilePath.getFileName()); + } + + private static Path getTmpCompressedTarFilePath(Path tarFilePath) { + return tarFilePath.getParent().resolve("tmp_comp_" + tarFilePath.getFileName()); + } + + private void compressTarFile() throws IOException { + try (InputStream fis = Files.newInputStream(getTmpTarFilePath(tarFilePath)); + OutputStream fos = Files.newOutputStream(getTmpCompressedTarFilePath(tarFilePath), StandardOpenOption.CREATE); + OutputStream compressedOS = getCompressedOutputStream(fos, this.compressionFormat)) { + byte[] buffer = new byte[8192]; + int len; + while ((len = fis.read(buffer)) != -1) { + compressedOS.write(buffer, 0, len); + } + } + } + + private static OutputStream getCompressedOutputStream(OutputStream os, CompressionFormat compressionFormat) throws IOException { + return compressionFormat == null ? os : switch (compressionFormat) { + case GZIP -> new GzipCompressorOutputStream(os); + case BZIP2 -> new BZip2CompressorOutputStream(os); + case XZ -> new XZCompressorOutputStream(os); + case ZSTD -> new ZstdCompressorOutputStream(os); + default -> os; + }; + } + + @Override + public void close() throws IOException { + if (!closed) { + + // Close temporary stream file + super.close(); + + // Open a new temporary archive + try (TarArchiveOutputStream taos = getTarStream(getTmpTarFilePath(tarFilePath))) { + + // Useful parameter + taos.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); + taos.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + + // Temporary stream file path + Path tmpStreamFilePath = getTmpStreamFilePath(tarFilePath); + + // Copy content of temporary stream file into an entry of the temporary archive + try (InputStream is = Files.newInputStream(tmpStreamFilePath)) { + + // New tar entry + TarArchiveEntry entry = new TarArchiveEntry(fileName); + entry.setSize(Files.size(tmpStreamFilePath)); + + // New file to add + taos.putArchiveEntry(entry); + + // Write the data in the entry + ByteStreams.copy(is, taos); + + // close new entry + taos.closeArchiveEntry(); + } + + // Copy existing entries into the temporary archive + if (Files.exists(tarFilePath)) { + try (InputStream fis = Files.newInputStream(tarFilePath); + BufferedInputStream bis = new BufferedInputStream(fis); + InputStream cis = getCompressedInputStream(bis, compressionFormat); + TarArchiveInputStream tarInput = new TarArchiveInputStream(cis)) { + TarArchiveEntry oldEntry; + while ((oldEntry = tarInput.getNextEntry()) != null) { + if (!oldEntry.getName().equals(fileName)) { + taos.putArchiveEntry(oldEntry); + byte[] buffer = new byte[8192]; + int len; + while ((len = tarInput.read(buffer)) != -1) { + taos.write(buffer, 0, len); + } + taos.closeArchiveEntry(); + } + } + } + } + + // Finishes the TAR archive without closing the underlying OutputStream + taos.finish(); + } + + // Compress the archive if needed + compressTarFile(); + + // swap with tmp tar + Path tmpTarFilePath = getTmpCompressedTarFilePath(tarFilePath); + Files.move(tmpTarFilePath, tarFilePath, StandardCopyOption.REPLACE_EXISTING); + + closed = true; + } + } + } + + private static InputStream getCompressedInputStream(InputStream is, CompressionFormat compressionFormat) throws IOException { + if (compressionFormat == null) { + return is; + } + return switch (compressionFormat) { + case GZIP -> new GzipCompressorInputStream(is); + case BZIP2 -> new BZip2CompressorInputStream(is); + case XZ -> new XZCompressorInputStream(is); + case ZSTD -> new ZstdCompressorInputStream(is); + default -> is; + }; + } +} diff --git a/commons/src/main/java/com/powsybl/commons/datasource/ZipArchiveDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/ZipArchiveDataSource.java index b6c90f26c19..8372a0fe63e 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/ZipArchiveDataSource.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/ZipArchiveDataSource.java @@ -61,7 +61,7 @@ public ZipArchiveDataSource(Path zipFile) { this(zipFile.getParent(), com.google.common.io.Files.getNameWithoutExtension(zipFile.getFileName().toString())); } - private static boolean entryExists(Path zipFilePath, String fileName) { + protected boolean entryExists(Path zipFilePath, String fileName) { if (Files.exists(zipFilePath)) { try (ZipFile zipFile = ZipFile.builder() .setSeekableByteChannel(Files.newByteChannel(zipFilePath)) @@ -74,13 +74,6 @@ private static boolean entryExists(Path zipFilePath, String fileName) { return false; } - @Override - public boolean exists(String fileName) { - Objects.requireNonNull(fileName); - Path zipFilePath = getArchiveFilePath(); - return entryExists(zipFilePath, fileName); - } - @Override public InputStream newInputStream(String suffix, String ext) throws IOException { return newInputStream(DataSourceUtil.getFileName(baseName, suffix, ext)); @@ -193,7 +186,6 @@ public OutputStream newOutputStream(String suffix, String ext, boolean append) t @Override public Set listNames(String regex) throws IOException { - // Consider only files in the given folder, do not go into folders Pattern p = Pattern.compile(regex); Set names = new HashSet<>(); Path zipFilePath = getArchiveFilePath(); diff --git a/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java b/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java index 90c27ac425b..872e93a9634 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java @@ -15,6 +15,7 @@ import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; @@ -47,6 +48,18 @@ void tearDown() throws Exception { fileSystem.close(); } + protected void checkDataSource(AbstractArchiveDataSource dataSource, String zipFileName, String baseName, + String dataExtension, ArchiveFormat archiveFormat, + CompressionFormat compressionFormat, DataSourceObserver observer) { + assertEquals(testDir, dataSource.getDirectory()); + assertEquals(zipFileName, dataSource.getArchiveFilePath().getFileName().toString()); + assertEquals(baseName, dataSource.getBaseName()); + assertEquals(dataExtension, dataSource.getDataExtension()); + assertEquals(archiveFormat, dataSource.getArchiveFormat()); + assertEquals(compressionFormat, dataSource.getCompressionFormat()); + assertEquals(observer, dataSource.getObserver()); + } + @Test void testFileInSubfolder() throws IOException { // File @@ -58,11 +71,33 @@ void testFileInSubfolder() throws IOException { // All the files are listed, no filter is applied Set files = dataSource.listNames(".*"); - assertEquals(3, files.size()); + assertEquals(4, files.size()); assertTrue(files.contains("foo.iidm")); assertTrue(files.contains("foo_bar.iidm")); assertFalse(files.contains("foo_baz.iidm")); assertTrue(files.contains("subfolder/foo_baz.iidm")); + assertTrue(files.contains("subfolder/subsubfolder/foo.v3.iidm")); + } + + @Test + void testStreams() throws IOException { + // Create file + createFiles(archiveWithSubfolders); + Path path = fileSystem.getPath(archiveWithSubfolders); + + // Datasource with an observer + DataSourceObserver observer = new DefaultDataSourceObserver(); + DataSource dataSource = DataSourceUtil.createDataSource(fileSystem.getPath("."), archiveWithSubfolders, observer); + assertInstanceOf(ObservableInputStream.class, dataSource.newInputStream("foo.iidm")); + assertInstanceOf(ObservableOutputStream.class, dataSource.newOutputStream("test.iidm", false)); + + // Datasource without an observer + dataSource = DataSource.fromPath(path); + try (InputStream inputStream = dataSource.newInputStream("foo.iidm"); + OutputStream outputStream = dataSource.newOutputStream("test.iidm", false)) { + assertFalse(inputStream instanceof ObservableInputStream); + assertFalse(outputStream instanceof ObservableOutputStream); + } } @Test @@ -81,4 +116,20 @@ void testErrorOnAppend() throws IOException { }); assertEquals(appendException, exception.getMessage()); } + + protected abstract AbstractArchiveDataSource createArchiveDataSource(); + + @Test + void testMissingArchive() throws IOException { + AbstractArchiveDataSource dataSource = createArchiveDataSource(); + assertFalse(dataSource.exists("test.bar")); + assertNull(dataSource.newInputStream("test.bar")); + } + + @Test + void testWrongTypeOfFile() throws IOException { + Files.createFile(testDir.resolve("foo.bar")); + AbstractArchiveDataSource dataSource = createArchiveDataSource(); + assertFalse(dataSource.exists("test.bar")); + } } diff --git a/commons/src/test/java/com/powsybl/commons/datasource/ArchiveFormatTest.java b/commons/src/test/java/com/powsybl/commons/datasource/ArchiveFormatTest.java index 0af78199448..a32e6a5d0cf 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/ArchiveFormatTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/ArchiveFormatTest.java @@ -19,11 +19,13 @@ class ArchiveFormatTest { @Test void test() { - assertEquals(1, ArchiveFormat.values().length); + assertEquals(2, ArchiveFormat.values().length); assertEquals("zip", ArchiveFormat.ZIP.getExtension()); + assertEquals("tar", ArchiveFormat.TAR.getExtension()); List formats = List.of( - ArchiveFormat.ZIP.name()); + ArchiveFormat.ZIP.name(), + ArchiveFormat.TAR.name()); assertEquals(formats, ArchiveFormat.getFormats()); } } diff --git a/commons/src/test/java/com/powsybl/commons/datasource/DataSourceBuilderTest.java b/commons/src/test/java/com/powsybl/commons/datasource/DataSourceBuilderTest.java index 1ad9ecfd6c3..c19aebbf6e5 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/DataSourceBuilderTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/DataSourceBuilderTest.java @@ -54,6 +54,7 @@ void testBuilder() { assertInstanceOf(Bzip2DirectoryDataSource.class, builder.withCompressionFormat(CompressionFormat.BZIP2).build()); // Archive datasources + assertInstanceOf(TarArchiveDataSource.class, builder.withArchiveFormat(ArchiveFormat.TAR).build()); assertInstanceOf(ZipArchiveDataSource.class, builder.withArchiveFormat(ArchiveFormat.ZIP).withCompressionFormat(null).build()); assertInstanceOf(ZipArchiveDataSource.class, builder.withCompressionFormat(CompressionFormat.ZIP).build()); assertInstanceOf(ZipArchiveDataSource.class, builder.withArchiveFormat(null).build()); @@ -96,9 +97,14 @@ void testBuilderErrorsZip() { .withArchiveFileName("bar.zip") .withDataExtension(".baz"); + // Wrong archive format + builder.withCompressionFormat(CompressionFormat.ZIP).withArchiveFormat(ArchiveFormat.TAR); + PowsyblException exception = assertThrows(PowsyblException.class, builder::build); + assertEquals("Incoherence between compression format ZIP and archive format TAR", exception.getMessage()); + // Wrong compression format builder.withCompressionFormat(CompressionFormat.GZIP).withArchiveFormat(ArchiveFormat.ZIP); - PowsyblException exception = assertThrows(PowsyblException.class, builder::build); + exception = assertThrows(PowsyblException.class, builder::build); assertEquals("Incoherence between compression format GZIP and archive format ZIP", exception.getMessage()); } } diff --git a/commons/src/test/java/com/powsybl/commons/datasource/FileInformationTest.java b/commons/src/test/java/com/powsybl/commons/datasource/FileInformationTest.java index 4396afc5586..d0a05c604e5 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/FileInformationTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/FileInformationTest.java @@ -22,6 +22,7 @@ class FileInformationTest { void tests() { unitTest("dummy", "dummy", null, null, ""); unitTest("dummy.iidm", "dummy", null, null, "iidm"); + unitTest("dummy.tar.gz", "dummy", CompressionFormat.GZIP, ArchiveFormat.TAR, ""); unitTest("dummy.xml.xz", "dummy", CompressionFormat.XZ, null, "xml"); // A zip file is a compressed archive @@ -40,6 +41,8 @@ void tests() { unitTest(".dummy", ".dummy", null, null, ""); unitTest(".iidm", ".iidm", null, null, ""); unitTest(".zip", "", CompressionFormat.ZIP, ArchiveFormat.ZIP, ""); + unitTest(".dummy.tar.gz", ".dummy", CompressionFormat.GZIP, ArchiveFormat.TAR, ""); + unitTest(".tar.gz", "", CompressionFormat.GZIP, ArchiveFormat.TAR, ""); unitTest(".dummy.jiidm.zip", ".dummy", CompressionFormat.ZIP, ArchiveFormat.ZIP, "jiidm"); PowsyblException exception = assertThrows(PowsyblException.class, () -> new FileInformation("")); diff --git a/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java b/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java new file mode 100644 index 00000000000..6c4ebca3f22 --- /dev/null +++ b/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * SPDX-License-Identifier: MPL-2.0 + */ +package com.powsybl.commons.datasource; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; +import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.provider.Arguments; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.Set; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author Nicolas Rol {@literal } + */ +class TarArchiveDataSourceTest extends AbstractArchiveDataSourceTest { + + private static final String WORK_DIR = "/work/"; + private static final String MAIN_EXT = "xml"; + private static final String BASENAME = "network"; + private static final String MAIN_FILE = BASENAME + "." + MAIN_EXT; + private static final String TAR_FILENAME = MAIN_FILE + ".tar.gz"; + private static final String TAR_PATH = WORK_DIR + TAR_FILENAME; + private static final String ADDITIONAL_SUFFIX = "_mapping"; + private static final String ADDITIONAL_EXT = "csv"; + private static final String ADDITIONAL_FILE = BASENAME + ADDITIONAL_SUFFIX + "." + ADDITIONAL_EXT; + private static final String UNRELATED_FILE = "other.de"; + + @Override + @BeforeEach + void setUp() throws Exception { + super.setUp(); + archiveWithSubfolders = "foo.iidm.tar.gz"; + appendException = "append not supported in tar file data source"; + archiveFormat = ArchiveFormat.TAR; + compressionFormat = CompressionFormat.GZIP; + } + + @Override + protected String getFileName(String baseName, String dataExtension, CompressionFormat compressionFormat) { + return testDir + "/" + baseName + + (dataExtension == null || dataExtension.isEmpty() ? "" : "." + dataExtension) + + (archiveFormat == null ? "" : "." + archiveFormat.getExtension()) + + (compressionFormat == null ? "" : "." + compressionFormat.getExtension()); + } + + @Test + @Override + void testConstructors() { + // Observer + DataSourceObserver observer = new DefaultDataSourceObserver(); + + // Check constructors + checkDataSource(new TarArchiveDataSource(testDir, "foo_bar.tar.gz", "foo", "iidm", compressionFormat, observer), "foo_bar.tar.gz", "foo", "iidm", archiveFormat, compressionFormat, observer); + checkDataSource(new TarArchiveDataSource(testDir, "foo_bar.tar.gz", "foo", "iidm", compressionFormat), "foo_bar.tar.gz", "foo", "iidm", archiveFormat, compressionFormat, null); + checkDataSource(new TarArchiveDataSource(testDir, "foo", "iidm", compressionFormat, observer), "foo.iidm.tar.gz", "foo", "iidm", archiveFormat, compressionFormat, observer); + checkDataSource(new TarArchiveDataSource(testDir, "foo", "", compressionFormat, observer), "foo.tar.gz", "foo", "", archiveFormat, compressionFormat, observer); + checkDataSource(new TarArchiveDataSource(testDir, "foo", "iidm", compressionFormat), "foo.iidm.tar.gz", "foo", "iidm", archiveFormat, compressionFormat, null); + checkDataSource(new TarArchiveDataSource(testDir, "foo", null, compressionFormat), "foo.tar.gz", "foo", null, archiveFormat, compressionFormat, null); + checkDataSource(new TarArchiveDataSource(testDir, "foo", "", compressionFormat), "foo.tar.gz", "foo", "", archiveFormat, compressionFormat, null); + checkDataSource(new TarArchiveDataSource(testDir, "foo", compressionFormat, observer), "foo.tar.gz", "foo", null, archiveFormat, compressionFormat, observer); + checkDataSource(new TarArchiveDataSource(testDir, "foo", compressionFormat), "foo.tar.gz", "foo", null, archiveFormat, compressionFormat, null); + checkDataSource(new TarArchiveDataSource(testDir, "foo"), "foo.tar", "foo", null, archiveFormat, null, null); + checkDataSource(new TarArchiveDataSource(testDir.resolve("foo_bar.tar.gz")), "foo_bar.tar.gz", "foo_bar", "", archiveFormat, compressionFormat, null); + } + + @Override + protected boolean appendTest() { + // append does not work with tar files + return false; + } + + @Override + protected DataSource createDataSource() { + return new TarArchiveDataSource(testDir, "foo.tar.gz", "foo", null, compressionFormat, null); + } + + @Override + protected DataSource createDataSource(DataSourceObserver observer) { + return new TarArchiveDataSource(testDir, "foo", "iidm", compressionFormat, observer); + } + + @Override + protected AbstractArchiveDataSource createArchiveDataSource() { + return new TarArchiveDataSource(testDir, "foo.bar", "foo", null, compressionFormat, null); + } + + static Stream provideArgumentsForWriteThenReadTest() { + return Stream.of( + Arguments.of("foo", "iidm", CompressionFormat.GZIP), + Arguments.of("foo", "", CompressionFormat.XZ), + Arguments.of("foo", "v3", CompressionFormat.ZSTD), + Arguments.of("foo", "v3", CompressionFormat.BZIP2), + Arguments.of("foo", "v3", null) + ); + } + + static Stream provideArgumentsForClassAndListingTest() { + return Stream.of( + Arguments.of("foo", "iidm", CompressionFormat.GZIP, TarArchiveDataSource.class, + Set.of("foo", "foo.txt", "foo.iidm", "foo.xiidm", "foo.v3.iidm", "foo.v3", "foo_bar.iidm", "foo_bar", "bar.iidm", "bar"), + Set.of("foo_bar.iidm", "foo_bar", "bar.iidm", "bar")), + Arguments.of("foo", "", CompressionFormat.BZIP2, TarArchiveDataSource.class, + Set.of("foo", "foo.txt", "foo.iidm", "foo.xiidm", "foo.v3.iidm", "foo.v3", "foo_bar.iidm", "foo_bar", "bar.iidm", "bar"), + Set.of("foo_bar.iidm", "foo_bar", "bar.iidm", "bar")), + Arguments.of("foo", "v3", CompressionFormat.ZSTD, TarArchiveDataSource.class, + Set.of("foo", "foo.txt", "foo.iidm", "foo.xiidm", "foo.v3.iidm", "foo.v3", "foo_bar.iidm", "foo_bar", "bar.iidm", "bar"), + Set.of("foo_bar.iidm", "foo_bar", "bar.iidm", "bar")) + ); + } + + @Override + protected void createFiles(String fileName) throws IOException { + + // File information + FileInformation fileInformation = new FileInformation(fileName); + + // Create the Tar archive and add the files + try (OutputStream fOut = Files.newOutputStream(fileSystem.getPath(fileName)); + BufferedOutputStream buffOut = new BufferedOutputStream(fOut); + OutputStream gzOut = getCompressedOutputStream(buffOut, fileInformation.getCompressionFormat()); + TarArchiveOutputStream tOut = new TarArchiveOutputStream(gzOut)) { + filesInArchive.forEach(fileInArchive -> { + try { + TarArchiveEntry e = new TarArchiveEntry(fileInArchive); + e.setSize(11); + tOut.putArchiveEntry(e); + byte[] data = "Test String".getBytes(); + tOut.write(data, 0, data.length); + tOut.closeArchiveEntry(); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }); + tOut.finish(); + } + } + + @Test + void fakeTarTest() throws IOException { + Files.createFile(testDir.resolve("fake.tar")); + assertFalse(new TarArchiveDataSource(testDir, "fake", null).exists("e")); + } + + @Test + void testTarDataSourceWithMoreThanOneDot() throws IOException { + // File information + FileInformation fileInformation = new FileInformation(TAR_PATH); + + // Create the Tar archive + try (OutputStream fOut = Files.newOutputStream(fileSystem.getPath(TAR_PATH)); + BufferedOutputStream buffOut = new BufferedOutputStream(fOut); + OutputStream gzOut = getCompressedOutputStream(buffOut, fileInformation.getCompressionFormat()); + TarArchiveOutputStream tOut = new TarArchiveOutputStream(gzOut)) { + // First entry + TarArchiveEntry e = new TarArchiveEntry(UNRELATED_FILE); + e.setSize(11); + tOut.putArchiveEntry(e); + byte[] data = "Test String".getBytes(); + tOut.write(data, 0, data.length); + tOut.closeArchiveEntry(); + + // Another entry + e = new TarArchiveEntry(MAIN_FILE); + e.setSize(13); + tOut.putArchiveEntry(e); + data = "Test String 2".getBytes(); + tOut.write(data, 0, data.length); + tOut.closeArchiveEntry(); + + // A third one + e = new TarArchiveEntry(ADDITIONAL_FILE); + e.setSize(13); + tOut.putArchiveEntry(e); + data = "Test String 2".getBytes(); + tOut.write(data, 0, data.length); + tOut.closeArchiveEntry(); + } + + // Create the datasource + var workdirPath = fileSystem.getPath(WORK_DIR); + DataSource dataSource = DataSourceUtil.createDataSource(workdirPath, TAR_FILENAME, null); + + // Assertions on the files in the archive + assertTrue(dataSource.exists(UNRELATED_FILE)); + assertFalse(dataSource.exists("not.tar.gz")); + assertTrue(dataSource.exists(null, MAIN_EXT)); + assertTrue(dataSource.exists(ADDITIONAL_SUFFIX, ADDITIONAL_EXT)); + assertFalse(dataSource.exists("-not", "there")); + try (InputStream is = dataSource.newInputStream(UNRELATED_FILE)) { + assertEquals("Test String", new String(is.readAllBytes())); + } + } + + private OutputStream getCompressedOutputStream(OutputStream os, CompressionFormat compressionFormat) throws IOException { + return compressionFormat == null ? os : switch (compressionFormat) { + case GZIP -> new GzipCompressorOutputStream(os); + case BZIP2 -> new BZip2CompressorOutputStream(os); + case XZ -> new XZCompressorOutputStream(os); + case ZSTD -> new ZstdCompressorOutputStream(os); + default -> os; + }; + } +} diff --git a/commons/src/test/java/com/powsybl/commons/datasource/ZipArchiveDataSourceTest.java b/commons/src/test/java/com/powsybl/commons/datasource/ZipArchiveDataSourceTest.java index f715be6551b..582f541c04f 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/ZipArchiveDataSourceTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/ZipArchiveDataSourceTest.java @@ -55,24 +55,16 @@ void testConstructors() { DataSourceObserver observer = new DefaultDataSourceObserver(); // Check constructors - checkDataSource(new ZipArchiveDataSource(testDir, "foo_bar.zip", "foo", "iidm", observer), "foo_bar.zip", "foo", "iidm", observer); - checkDataSource(new ZipArchiveDataSource(testDir, "foo_bar.zip", "foo", "iidm"), "foo_bar.zip", "foo", "iidm", null); - checkDataSource(new ZipArchiveDataSource(testDir, "foo", "iidm", observer), "foo.iidm.zip", "foo", "iidm", observer); - checkDataSource(new ZipArchiveDataSource(testDir, "foo", "", observer), "foo.zip", "foo", "", observer); - checkDataSource(new ZipArchiveDataSource(testDir, "foo", "iidm"), "foo.iidm.zip", "foo", "iidm", null); - checkDataSource(new ZipArchiveDataSource(testDir, "foo", (String) null), "foo.zip", "foo", null, null); - checkDataSource(new ZipArchiveDataSource(testDir, "foo", ""), "foo.zip", "foo", "", null); - checkDataSource(new ZipArchiveDataSource(testDir, "foo", observer), "foo.zip", "foo", null, observer); - checkDataSource(new ZipArchiveDataSource(testDir, "foo"), "foo.zip", "foo", null, null); - checkDataSource(new ZipArchiveDataSource(testDir.resolve("foo_bar.zip")), "foo_bar.zip", "foo_bar", null, null); - } - - private void checkDataSource(ZipArchiveDataSource dataSource, String zipFileName, String baseName, String dataExtension, DataSourceObserver observer) { - assertEquals(testDir, dataSource.getDirectory()); - assertEquals(dataExtension, dataSource.getDataExtension()); - assertEquals(zipFileName, dataSource.getArchiveFilePath().getFileName().toString()); - assertEquals(baseName, dataSource.getBaseName()); - assertEquals(observer, dataSource.getObserver()); + checkDataSource(new ZipArchiveDataSource(testDir, "foo_bar.zip", "foo", "iidm", observer), "foo_bar.zip", "foo", "iidm", archiveFormat, compressionFormat, observer); + checkDataSource(new ZipArchiveDataSource(testDir, "foo_bar.zip", "foo", "iidm"), "foo_bar.zip", "foo", "iidm", archiveFormat, compressionFormat, null); + checkDataSource(new ZipArchiveDataSource(testDir, "foo", "iidm", observer), "foo.iidm.zip", "foo", "iidm", archiveFormat, compressionFormat, observer); + checkDataSource(new ZipArchiveDataSource(testDir, "foo", "", observer), "foo.zip", "foo", "", archiveFormat, compressionFormat, observer); + checkDataSource(new ZipArchiveDataSource(testDir, "foo", "iidm"), "foo.iidm.zip", "foo", "iidm", archiveFormat, compressionFormat, null); + checkDataSource(new ZipArchiveDataSource(testDir, "foo", (String) null), "foo.zip", "foo", null, archiveFormat, compressionFormat, null); + checkDataSource(new ZipArchiveDataSource(testDir, "foo", ""), "foo.zip", "foo", "", archiveFormat, compressionFormat, null); + checkDataSource(new ZipArchiveDataSource(testDir, "foo", observer), "foo.zip", "foo", null, archiveFormat, compressionFormat, observer); + checkDataSource(new ZipArchiveDataSource(testDir, "foo"), "foo.zip", "foo", null, archiveFormat, compressionFormat, null); + checkDataSource(new ZipArchiveDataSource(testDir.resolve("foo_bar.zip")), "foo_bar.zip", "foo_bar", null, archiveFormat, compressionFormat, null); } @Override @@ -90,6 +82,11 @@ protected DataSource createDataSource(DataSourceObserver observer) { return new ZipArchiveDataSource(testDir, "foo", "iidm", observer); } + @Override + protected AbstractArchiveDataSource createArchiveDataSource() { + return new ZipArchiveDataSource(testDir, "foo.bar", "foo", null, null); + } + static Stream provideArgumentsForWriteThenReadTest() { return Stream.of( Arguments.of("foo", "iidm", CompressionFormat.ZIP), @@ -161,8 +158,12 @@ void createZipDataSourceWithMoreThanOneDot() throws IOException { out.closeEntry(); } + + // Create the datasource var workdirPath = fileSystem.getPath(WORK_DIR); DataSource dataSource = DataSourceUtil.createDataSource(workdirPath, ZIP_FILENAME, null); + + // Assertions on the files in the archive assertTrue(dataSource.exists(UNRELATED_FILE)); assertFalse(dataSource.exists("not.zip")); assertTrue(dataSource.exists(null, MAIN_EXT)); diff --git a/commons/src/test/resources/foo.iidm.tar.gz b/commons/src/test/resources/foo.iidm.tar.gz new file mode 100644 index 00000000000..12f886ea42b Binary files /dev/null and b/commons/src/test/resources/foo.iidm.tar.gz differ diff --git a/commons/src/test/resources/foo.iidm.zip b/commons/src/test/resources/foo.iidm.zip index 37c5709db71..acc26ee76a4 100644 Binary files a/commons/src/test/resources/foo.iidm.zip and b/commons/src/test/resources/foo.iidm.zip differ