From 8e2c706989187cd318e8b9b04f907f5ee1114163 Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Wed, 17 Jul 2024 11:46:19 +0200 Subject: [PATCH 01/12] add TarArchiveDataSource Signed-off-by: Nicolas Rol --- .../datasource/AbstractArchiveDataSource.java | 14 + .../commons/datasource/ArchiveFormat.java | 3 +- .../commons/datasource/DataSourceBuilder.java | 4 + .../commons/datasource/FileInformation.java | 12 +- .../datasource/TarArchiveDataSource.java | 316 ++++++++++++++++++ .../datasource/ZipArchiveDataSource.java | 9 +- .../AbstractArchiveDataSourceTest.java | 12 + .../commons/datasource/ArchiveFormatTest.java | 6 +- .../datasource/DataSourceBuilderTest.java | 8 +- .../datasource/FileInformationTest.java | 3 + .../datasource/TarArchiveDataSourceTest.java | 205 ++++++++++++ .../datasource/ZipArchiveDataSourceTest.java | 32 +- commons/src/test/resources/foo.iidm.tar.gz | Bin 0 -> 230 bytes 13 files changed, 592 insertions(+), 32 deletions(-) create mode 100644 commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java create mode 100644 commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java create mode 100644 commons/src/test/resources/foo.iidm.tar.gz diff --git a/commons/src/main/java/com/powsybl/commons/datasource/AbstractArchiveDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/AbstractArchiveDataSource.java index 6871f2b1149..4e2bc0bcbe0 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/AbstractArchiveDataSource.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/AbstractArchiveDataSource.java @@ -8,6 +8,7 @@ package com.powsybl.commons.datasource; import java.nio.file.Path; +import java.util.Objects; /** * @author Nicolas Rol {@literal } @@ -26,4 +27,17 @@ public abstract class AbstractArchiveDataSource extends AbstractFileSystemDataSo protected Path getArchiveFilePath() { return directory.resolve(archiveFileName); } + + protected ArchiveFormat getArchiveFormat() { + return archiveFormat; + } + + protected abstract boolean entryExists(Path archiveFilePath, String fileName); + + @Override + public boolean exists(String fileName) { + Objects.requireNonNull(fileName); + Path archiveFilePath = getArchiveFilePath(); + return entryExists(archiveFilePath, fileName); + } } diff --git a/commons/src/main/java/com/powsybl/commons/datasource/ArchiveFormat.java b/commons/src/main/java/com/powsybl/commons/datasource/ArchiveFormat.java index 8de2889c47a..c9bfe975c93 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/ArchiveFormat.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/ArchiveFormat.java @@ -15,7 +15,8 @@ * @author Nicolas Rol {@literal } */ public enum ArchiveFormat { - ZIP("zip"); + ZIP("zip"), + TAR("tar"); ArchiveFormat(String extension) { this.extension = Objects.requireNonNull(extension); diff --git a/commons/src/main/java/com/powsybl/commons/datasource/DataSourceBuilder.java b/commons/src/main/java/com/powsybl/commons/datasource/DataSourceBuilder.java index 9736c8d3cbe..c3f4746e145 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/DataSourceBuilder.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/DataSourceBuilder.java @@ -69,6 +69,10 @@ DataSource build() { // Create the datasource if (compressionFormat == CompressionFormat.ZIP || archiveFormat == ArchiveFormat.ZIP) { return buildZip(); + } else if (archiveFormat == ArchiveFormat.TAR) { + return archiveFileName == null ? + new TarArchiveDataSource(directory, baseName, dataExtension, compressionFormat, observer) : + new TarArchiveDataSource(directory, archiveFileName, baseName, dataExtension, compressionFormat, observer); } else if (compressionFormat == null) { return new DirectoryDataSource(directory, baseName, dataExtension, observer); } else { diff --git a/commons/src/main/java/com/powsybl/commons/datasource/FileInformation.java b/commons/src/main/java/com/powsybl/commons/datasource/FileInformation.java index 2cb17ece723..2de0d0bae75 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/FileInformation.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/FileInformation.java @@ -56,10 +56,18 @@ private void computeInformation(String fileName, boolean dataSourceInitializatio // File name without the compression extension String fileNameWithoutCompressionExtension = compressionFormat == null ? fileName : fileName.substring(0, currentDotIndex); + // Last dot index + currentDotIndex = fileNameWithoutCompressionExtension.lastIndexOf('.'); + // Archive extension String fileNameWithoutCompressionNorArchive; - archiveFormat = compressionFormat == CompressionFormat.ZIP ? ArchiveFormat.ZIP : null; - fileNameWithoutCompressionNorArchive = fileNameWithoutCompressionExtension; + if (compressionFormat == CompressionFormat.ZIP) { + archiveFormat = ArchiveFormat.ZIP; + fileNameWithoutCompressionNorArchive = fileNameWithoutCompressionExtension; + } else { + archiveFormat = "tar".equals(fileNameWithoutCompressionExtension.substring(currentDotIndex + 1)) ? ArchiveFormat.TAR : null; + fileNameWithoutCompressionNorArchive = archiveFormat == null ? fileNameWithoutCompressionExtension : fileNameWithoutCompressionExtension.substring(0, currentDotIndex); + } // Last dot index currentDotIndex = fileNameWithoutCompressionNorArchive.lastIndexOf('.'); diff --git a/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java new file mode 100644 index 00000000000..ccce9e52997 --- /dev/null +++ b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * SPDX-License-Identifier: MPL-2.0 + */ +package com.powsybl.commons.datasource; + +import com.powsybl.commons.io.ForwardingOutputStream; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; +import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream; +import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * @author Nicolas Rol {@literal } + */ +public class TarArchiveDataSource extends AbstractArchiveDataSource { + + public TarArchiveDataSource(Path directory, String tarFileName, String baseName, String dataExtension, CompressionFormat compressionFormat, DataSourceObserver observer) { + super(directory, tarFileName, baseName, dataExtension, compressionFormat, ArchiveFormat.TAR, observer); + } + + public TarArchiveDataSource(Path directory, String tarFileName, String baseName, String dataExtension, CompressionFormat compressionFormat) { + this(directory, tarFileName, baseName, dataExtension, compressionFormat, null); + } + + public TarArchiveDataSource(Path directory, String baseName, String dataExtension, CompressionFormat compressionFormat, DataSourceObserver observer) { + this(directory, + baseName + (dataExtension == null || dataExtension.isEmpty() ? "" : "." + dataExtension) + ".tar" + (compressionFormat == null ? "" : "." + compressionFormat.getExtension()), + baseName, dataExtension, compressionFormat, observer); + } + + public TarArchiveDataSource(Path directory, String baseName, String dataExtension, CompressionFormat compressionFormat) { + this(directory, + baseName + (dataExtension == null || dataExtension.isEmpty() ? "" : "." + dataExtension) + ".tar" + (compressionFormat == null ? "" : "." + compressionFormat.getExtension()), + baseName, dataExtension, compressionFormat, null); + } + + public TarArchiveDataSource(Path directory, String baseName, CompressionFormat compressionFormat, DataSourceObserver observer) { + this(directory, + baseName + ".tar" + (compressionFormat == null ? "" : "." + compressionFormat.getExtension()), + baseName, null, compressionFormat, observer); + } + + public TarArchiveDataSource(Path directory, String baseName, CompressionFormat compressionFormat) { + this(directory, + baseName + ".tar" + (compressionFormat == null ? "" : "." + compressionFormat.getExtension()), + baseName, null, compressionFormat, null); + } + + public TarArchiveDataSource(Path directory, String baseName) { + this(directory, + baseName + ".tar", + baseName, null, null, null); + } + + public TarArchiveDataSource(Path tarFile) { + this(tarFile.getParent(), (new FileInformation(tarFile.getFileName().toString(), false)).getBaseName(), + (new FileInformation(tarFile.getFileName().toString(), false)).getDataExtension(), + (new FileInformation(tarFile.getFileName().toString(), false)).getCompressionFormat()); + } + + /** + * {@inheritDoc} + * + *

Files are here located in the archive.

+ */ + @Override + public Set listNames(String regex) throws IOException { + // Initialize variables + Pattern p = Pattern.compile(regex); + Set names = new HashSet<>(); + Path tarFilePath = getArchiveFilePath(); + + // Explore the archive + try (BufferedInputStream inputStream = new BufferedInputStream(Files.newInputStream(tarFilePath)); + InputStream cis = getCompressedInputStream(inputStream, compressionFormat); + TarArchiveInputStream tar = new TarArchiveInputStream(cis)) { + ArchiveEntry entry; + while ((entry = tar.getNextEntry()) != null) { + // Consider only files located at the root of the archive + if (!entry.isDirectory() + && entry.getName().indexOf('/') == entry.getName().lastIndexOf('/') + && p.matcher(entry.getName()).matches()) { + names.add(entry.getName()); + } + + } + } + return names; + } + + protected boolean entryExists(Path tarFilePath, String fileName) { + if (Files.exists(tarFilePath)) { + try (InputStream fis = Files.newInputStream(tarFilePath); + BufferedInputStream bis = new BufferedInputStream(fis); + InputStream is = getCompressedInputStream(bis, compressionFormat); + TarArchiveInputStream tais = new TarArchiveInputStream(is)) { + + TarArchiveEntry entry; + while ((entry = tais.getNextEntry()) != null) { + if (entry.getName().equals(fileName)) { + return true; + } + } + return false; + } catch (IOException | UnsupportedOperationException e) { + return false; + } + } + return false; + } + + @Override + public OutputStream newOutputStream(String suffix, String ext, boolean append) throws IOException { + return newOutputStream(DataSourceUtil.getFileName(baseName, suffix, ext), append); + } + + @Override + public OutputStream newOutputStream(String fileName, boolean append) throws IOException { + Objects.requireNonNull(fileName); + if (append) { + throw new UnsupportedOperationException("append not supported in tar file data source"); + } + Path tarFilePath = getArchiveFilePath(); + OutputStream os = new TarEntryOutputStream(tarFilePath, fileName, compressionFormat); + return observer != null ? new ObservableOutputStream(os, tarFilePath + ":" + fileName, observer) : os; + } + + @Override + public InputStream newInputStream(String suffix, String ext) throws IOException { + return newInputStream(DataSourceUtil.getFileName(baseName, suffix, ext)); + } + + @Override + public InputStream newInputStream(String fileName) throws IOException { + Objects.requireNonNull(fileName); + Path tarFilePath = getArchiveFilePath(); + try { + InputStream fis = Files.newInputStream(tarFilePath); + BufferedInputStream bis = new BufferedInputStream(fis); + InputStream is = getCompressedInputStream(bis, this.compressionFormat); + TarArchiveInputStream tais = new TarArchiveInputStream(is); + TarArchiveEntry entry; + while ((entry = tais.getNextEntry()) != null) { + if (entry.getName().equals(fileName)) { + return observer != null ? new ObservableInputStream(tais, tarFilePath + ":" + fileName, observer) : tais; + } + } + return null; + } catch (IOException e) { + return null; + } + } + + private static final class TarEntryOutputStream extends ForwardingOutputStream { + + private final Path tarFilePath; + private final String fileName; + private final CompressionFormat compressionFormat; + private boolean closed; + + private TarEntryOutputStream(Path tarFilePath, String fileName, CompressionFormat compressionFormat) throws IOException { + super(getTmpStream(getTmpStreamFilePath(tarFilePath))); + this.tarFilePath = tarFilePath; + this.fileName = fileName; + this.compressionFormat = compressionFormat; + this.closed = false; + } + + private static OutputStream getTmpStream(Path tarFilePath) throws IOException { + return new BufferedOutputStream(Files.newOutputStream(tarFilePath)); + } + + private static Path getTmpStreamFilePath(Path tarFilePath) { + return tarFilePath.getParent().resolve("tmp_stream_" + tarFilePath.getFileName() + ".stream"); + } + + private static TarArchiveOutputStream getTarStream(Path tmpTarFilePath) throws IOException { + return new TarArchiveOutputStream(new BufferedOutputStream(Files.newOutputStream(tmpTarFilePath))); + } + + private static Path getTmpTarFilePath(Path tarFilePath) { + return tarFilePath.getParent().resolve("tmp_" + tarFilePath.getFileName()); + } + + private static Path getTmpCompressedTarFilePath(Path tarFilePath) { + return tarFilePath.getParent().resolve("tmp_comp_" + tarFilePath.getFileName()); + } + + private void compressTarFile() throws IOException { + try (InputStream fis = Files.newInputStream(getTmpTarFilePath(tarFilePath)); + OutputStream fos = Files.newOutputStream(getTmpCompressedTarFilePath(tarFilePath), StandardOpenOption.CREATE); + OutputStream compressedOS = getCompressedOutputStream(fos, this.compressionFormat)) { + byte[] buffer = new byte[8192]; + int len; + while ((len = fis.read(buffer)) != -1) { + compressedOS.write(buffer, 0, len); + } + } + } + + private static OutputStream getCompressedOutputStream(OutputStream is, CompressionFormat compressionFormat) throws IOException { + return switch (compressionFormat) { + case GZIP -> new GzipCompressorOutputStream(is); + case BZIP2 -> new BZip2CompressorOutputStream(is); + case XZ -> new XZCompressorOutputStream(is); + case ZSTD -> new ZstdCompressorOutputStream(is); + default -> is; + }; + } + + @Override + public void close() throws IOException { + if (!closed) { + + // Close temporary stream file + super.close(); + + // Open a new temporary archive + try (TarArchiveOutputStream taos = getTarStream(getTmpTarFilePath(tarFilePath))) { + + // Useful parameter + taos.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); + taos.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + + // Copy content of temporary stream file into an entry of the temporary archive + try (InputStream is = Files.newInputStream(getTmpStreamFilePath(tarFilePath))) { + + // Content of the stream + byte[] streamContent = is.readAllBytes(); + + // New tar entry + TarArchiveEntry entry = new TarArchiveEntry(fileName); + entry.setSize(streamContent.length); + + // New file to add + taos.putArchiveEntry(entry); + + // Write the data in the entry + taos.write(streamContent); + + // close new entry + taos.closeArchiveEntry(); + } + + // Copy existing entries into the temporary archive + if (Files.exists(tarFilePath)) { + try (InputStream fis = Files.newInputStream(tarFilePath); + BufferedInputStream bis = new BufferedInputStream(fis); + InputStream cis = getCompressedInputStream(bis, compressionFormat); + TarArchiveInputStream tarInput = new TarArchiveInputStream(cis)) { + TarArchiveEntry oldEntry; + while ((oldEntry = tarInput.getNextEntry()) != null) { + if (!oldEntry.getName().equals(fileName)) { + taos.putArchiveEntry(oldEntry); + byte[] buffer = new byte[8192]; + int len; + while ((len = tarInput.read(buffer)) != -1) { + taos.write(buffer, 0, len); + } + taos.closeArchiveEntry(); + } + } + } + } + + // Finishes the TAR archive without closing the underlying OutputStream + taos.finish(); + } + + // Compress the archive if needed + compressTarFile(); + + // swap with tmp tar + Path tmpTarFilePath = getTmpCompressedTarFilePath(tarFilePath); + Files.move(tmpTarFilePath, tarFilePath, StandardCopyOption.REPLACE_EXISTING); + + closed = true; + } + } + } + + private static InputStream getCompressedInputStream(InputStream is, CompressionFormat compressionFormat) throws IOException { + if (compressionFormat == null) { + return is; + } + return switch (compressionFormat) { + case GZIP -> new GzipCompressorInputStream(is); + case BZIP2 -> new BZip2CompressorInputStream(is); + case XZ -> new XZCompressorInputStream(is); + case ZSTD -> new ZstdCompressorInputStream(is); + default -> is; + }; + } +} diff --git a/commons/src/main/java/com/powsybl/commons/datasource/ZipArchiveDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/ZipArchiveDataSource.java index b6c90f26c19..8a63508c040 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/ZipArchiveDataSource.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/ZipArchiveDataSource.java @@ -61,7 +61,7 @@ public ZipArchiveDataSource(Path zipFile) { this(zipFile.getParent(), com.google.common.io.Files.getNameWithoutExtension(zipFile.getFileName().toString())); } - private static boolean entryExists(Path zipFilePath, String fileName) { + protected boolean entryExists(Path zipFilePath, String fileName) { if (Files.exists(zipFilePath)) { try (ZipFile zipFile = ZipFile.builder() .setSeekableByteChannel(Files.newByteChannel(zipFilePath)) @@ -74,13 +74,6 @@ private static boolean entryExists(Path zipFilePath, String fileName) { return false; } - @Override - public boolean exists(String fileName) { - Objects.requireNonNull(fileName); - Path zipFilePath = getArchiveFilePath(); - return entryExists(zipFilePath, fileName); - } - @Override public InputStream newInputStream(String suffix, String ext) throws IOException { return newInputStream(DataSourceUtil.getFileName(baseName, suffix, ext)); diff --git a/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java b/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java index 90c27ac425b..cb04da1c71b 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java @@ -47,6 +47,18 @@ void tearDown() throws Exception { fileSystem.close(); } + protected void checkDataSource(AbstractArchiveDataSource dataSource, String zipFileName, String baseName, + String dataExtension, ArchiveFormat archiveFormat, + CompressionFormat compressionFormat, DataSourceObserver observer) { + assertEquals(testDir, dataSource.getDirectory()); + assertEquals(zipFileName, dataSource.getArchiveFilePath().getFileName().toString()); + assertEquals(baseName, dataSource.getBaseName()); + assertEquals(dataExtension, dataSource.getDataExtension()); + assertEquals(archiveFormat, dataSource.getArchiveFormat()); + assertEquals(compressionFormat, dataSource.getCompressionFormat()); + assertEquals(observer, dataSource.getObserver()); + } + @Test void testFileInSubfolder() throws IOException { // File diff --git a/commons/src/test/java/com/powsybl/commons/datasource/ArchiveFormatTest.java b/commons/src/test/java/com/powsybl/commons/datasource/ArchiveFormatTest.java index 0af78199448..a32e6a5d0cf 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/ArchiveFormatTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/ArchiveFormatTest.java @@ -19,11 +19,13 @@ class ArchiveFormatTest { @Test void test() { - assertEquals(1, ArchiveFormat.values().length); + assertEquals(2, ArchiveFormat.values().length); assertEquals("zip", ArchiveFormat.ZIP.getExtension()); + assertEquals("tar", ArchiveFormat.TAR.getExtension()); List formats = List.of( - ArchiveFormat.ZIP.name()); + ArchiveFormat.ZIP.name(), + ArchiveFormat.TAR.name()); assertEquals(formats, ArchiveFormat.getFormats()); } } diff --git a/commons/src/test/java/com/powsybl/commons/datasource/DataSourceBuilderTest.java b/commons/src/test/java/com/powsybl/commons/datasource/DataSourceBuilderTest.java index 1ad9ecfd6c3..c19aebbf6e5 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/DataSourceBuilderTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/DataSourceBuilderTest.java @@ -54,6 +54,7 @@ void testBuilder() { assertInstanceOf(Bzip2DirectoryDataSource.class, builder.withCompressionFormat(CompressionFormat.BZIP2).build()); // Archive datasources + assertInstanceOf(TarArchiveDataSource.class, builder.withArchiveFormat(ArchiveFormat.TAR).build()); assertInstanceOf(ZipArchiveDataSource.class, builder.withArchiveFormat(ArchiveFormat.ZIP).withCompressionFormat(null).build()); assertInstanceOf(ZipArchiveDataSource.class, builder.withCompressionFormat(CompressionFormat.ZIP).build()); assertInstanceOf(ZipArchiveDataSource.class, builder.withArchiveFormat(null).build()); @@ -96,9 +97,14 @@ void testBuilderErrorsZip() { .withArchiveFileName("bar.zip") .withDataExtension(".baz"); + // Wrong archive format + builder.withCompressionFormat(CompressionFormat.ZIP).withArchiveFormat(ArchiveFormat.TAR); + PowsyblException exception = assertThrows(PowsyblException.class, builder::build); + assertEquals("Incoherence between compression format ZIP and archive format TAR", exception.getMessage()); + // Wrong compression format builder.withCompressionFormat(CompressionFormat.GZIP).withArchiveFormat(ArchiveFormat.ZIP); - PowsyblException exception = assertThrows(PowsyblException.class, builder::build); + exception = assertThrows(PowsyblException.class, builder::build); assertEquals("Incoherence between compression format GZIP and archive format ZIP", exception.getMessage()); } } diff --git a/commons/src/test/java/com/powsybl/commons/datasource/FileInformationTest.java b/commons/src/test/java/com/powsybl/commons/datasource/FileInformationTest.java index 4396afc5586..d0a05c604e5 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/FileInformationTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/FileInformationTest.java @@ -22,6 +22,7 @@ class FileInformationTest { void tests() { unitTest("dummy", "dummy", null, null, ""); unitTest("dummy.iidm", "dummy", null, null, "iidm"); + unitTest("dummy.tar.gz", "dummy", CompressionFormat.GZIP, ArchiveFormat.TAR, ""); unitTest("dummy.xml.xz", "dummy", CompressionFormat.XZ, null, "xml"); // A zip file is a compressed archive @@ -40,6 +41,8 @@ void tests() { unitTest(".dummy", ".dummy", null, null, ""); unitTest(".iidm", ".iidm", null, null, ""); unitTest(".zip", "", CompressionFormat.ZIP, ArchiveFormat.ZIP, ""); + unitTest(".dummy.tar.gz", ".dummy", CompressionFormat.GZIP, ArchiveFormat.TAR, ""); + unitTest(".tar.gz", "", CompressionFormat.GZIP, ArchiveFormat.TAR, ""); unitTest(".dummy.jiidm.zip", ".dummy", CompressionFormat.ZIP, ArchiveFormat.ZIP, "jiidm"); PowsyblException exception = assertThrows(PowsyblException.class, () -> new FileInformation("")); diff --git a/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java b/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java new file mode 100644 index 00000000000..1c1a7989b58 --- /dev/null +++ b/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * SPDX-License-Identifier: MPL-2.0 + */ +package com.powsybl.commons.datasource; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; +import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.provider.Arguments; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.Set; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author Nicolas Rol {@literal } + */ +class TarArchiveDataSourceTest extends AbstractArchiveDataSourceTest { + + private static final String WORK_DIR = "/work/"; + private static final String MAIN_EXT = "xml"; + private static final String BASENAME = "network"; + private static final String MAIN_FILE = BASENAME + "." + MAIN_EXT; + private static final String TAR_FILENAME = MAIN_FILE + ".tar.gz"; + private static final String TAR_PATH = WORK_DIR + TAR_FILENAME; + private static final String ADDITIONAL_SUFFIX = "_mapping"; + private static final String ADDITIONAL_EXT = "csv"; + private static final String ADDITIONAL_FILE = BASENAME + ADDITIONAL_SUFFIX + "." + ADDITIONAL_EXT; + private static final String UNRELATED_FILE = "other.de"; + + @Override + @BeforeEach + void setUp() throws Exception { + super.setUp(); + archiveWithSubfolders = "foo.iidm.tar.gz"; + appendException = "append not supported in tar file data source"; + archiveFormat = ArchiveFormat.TAR; + compressionFormat = CompressionFormat.GZIP; + } + + @Test + @Override + void testConstructors() { + // Observer + DataSourceObserver observer = new DefaultDataSourceObserver(); + + // Check constructors + checkDataSource(new TarArchiveDataSource(testDir, "foo_bar.tar.gz", "foo", "iidm", compressionFormat, observer), "foo_bar.tar.gz", "foo", "iidm", archiveFormat, compressionFormat, observer); + checkDataSource(new TarArchiveDataSource(testDir, "foo_bar.tar.gz", "foo", "iidm", compressionFormat), "foo_bar.tar.gz", "foo", "iidm", archiveFormat, compressionFormat, null); + checkDataSource(new TarArchiveDataSource(testDir, "foo", "iidm", compressionFormat, observer), "foo.iidm.tar.gz", "foo", "iidm", archiveFormat, compressionFormat, observer); + checkDataSource(new TarArchiveDataSource(testDir, "foo", "", compressionFormat, observer), "foo.tar.gz", "foo", "", archiveFormat, compressionFormat, observer); + checkDataSource(new TarArchiveDataSource(testDir, "foo", "iidm", compressionFormat), "foo.iidm.tar.gz", "foo", "iidm", archiveFormat, compressionFormat, null); + checkDataSource(new TarArchiveDataSource(testDir, "foo", null, compressionFormat), "foo.tar.gz", "foo", null, archiveFormat, compressionFormat, null); + checkDataSource(new TarArchiveDataSource(testDir, "foo", "", compressionFormat), "foo.tar.gz", "foo", "", archiveFormat, compressionFormat, null); + checkDataSource(new TarArchiveDataSource(testDir, "foo", compressionFormat, observer), "foo.tar.gz", "foo", null, archiveFormat, compressionFormat, observer); + checkDataSource(new TarArchiveDataSource(testDir, "foo", compressionFormat), "foo.tar.gz", "foo", null, archiveFormat, compressionFormat, null); + checkDataSource(new TarArchiveDataSource(testDir.resolve("foo_bar.tar.gz")), "foo_bar.tar.gz", "foo_bar", "", archiveFormat, compressionFormat, null); + } + + @Override + protected boolean appendTest() { + // append does not work with tar files + return false; + } + + @Override + protected DataSource createDataSource() { + return new TarArchiveDataSource(testDir, "foo.tar.gz", "foo", null, compressionFormat, null); + } + + @Override + protected DataSource createDataSource(DataSourceObserver observer) { + return new TarArchiveDataSource(testDir, "foo", "iidm", compressionFormat, observer); + } + + static Stream provideArgumentsForWriteThenReadTest() { + return Stream.of( + Arguments.of("foo", "iidm", CompressionFormat.GZIP), + Arguments.of("foo", "", CompressionFormat.XZ), + Arguments.of("foo", "v3", CompressionFormat.ZSTD) + ); + } + + static Stream provideArgumentsForClassAndListingTest() { + return Stream.of( + Arguments.of("foo", "iidm", CompressionFormat.GZIP, TarArchiveDataSource.class, + Set.of("foo", "foo.txt", "foo.iidm", "foo.xiidm", "foo.v3.iidm", "foo.v3", "foo_bar.iidm", "foo_bar", "bar.iidm", "bar"), + Set.of("foo_bar.iidm", "foo_bar", "bar.iidm", "bar")), + Arguments.of("foo", "", CompressionFormat.BZIP2, TarArchiveDataSource.class, + Set.of("foo", "foo.txt", "foo.iidm", "foo.xiidm", "foo.v3.iidm", "foo.v3", "foo_bar.iidm", "foo_bar", "bar.iidm", "bar"), + Set.of("foo_bar.iidm", "foo_bar", "bar.iidm", "bar")), + Arguments.of("foo", "v3", CompressionFormat.ZSTD, TarArchiveDataSource.class, + Set.of("foo", "foo.txt", "foo.iidm", "foo.xiidm", "foo.v3.iidm", "foo.v3", "foo_bar.iidm", "foo_bar", "bar.iidm", "bar"), + Set.of("foo_bar.iidm", "foo_bar", "bar.iidm", "bar")) + ); + } + + @Override + protected void createArchiveAndFiles(String fileName) throws IOException { + + // File information + FileInformation fileInformation = new FileInformation(fileName); + + // Create the Tar archive and add the files + try (OutputStream fOut = Files.newOutputStream(fileSystem.getPath(fileName)); + BufferedOutputStream buffOut = new BufferedOutputStream(fOut); + OutputStream gzOut = getCompressedOutputStream(buffOut, fileInformation.getCompressionFormat()); + TarArchiveOutputStream tOut = new TarArchiveOutputStream(gzOut)) { + filesInArchive.forEach(fileInArchive -> { + try { + TarArchiveEntry e = new TarArchiveEntry(fileInArchive); + e.setSize(11); + tOut.putArchiveEntry(e); + byte[] data = "Test String".getBytes(); + tOut.write(data, 0, data.length); + tOut.closeArchiveEntry(); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }); + tOut.finish(); + } + } + + @Test + void fakeTarTest() throws IOException { + Files.createFile(testDir.resolve("fake.tar")); + assertFalse(new TarArchiveDataSource(testDir, "fake", null).exists("e")); + } + + @Test + void testTarDataSourceWithMoreThanOneDot() throws IOException { + // File information + FileInformation fileInformation = new FileInformation(TAR_PATH); + + // Create the Tar archive + try (OutputStream fOut = Files.newOutputStream(fileSystem.getPath(TAR_PATH)); + BufferedOutputStream buffOut = new BufferedOutputStream(fOut); + OutputStream gzOut = getCompressedOutputStream(buffOut, fileInformation.getCompressionFormat()); + TarArchiveOutputStream tOut = new TarArchiveOutputStream(gzOut)) { + // First entry + TarArchiveEntry e = new TarArchiveEntry(UNRELATED_FILE); + e.setSize(11); + tOut.putArchiveEntry(e); + byte[] data = "Test String".getBytes(); + tOut.write(data, 0, data.length); + tOut.closeArchiveEntry(); + + // Another entry + e = new TarArchiveEntry(MAIN_FILE); + e.setSize(13); + tOut.putArchiveEntry(e); + data = "Test String 2".getBytes(); + tOut.write(data, 0, data.length); + tOut.closeArchiveEntry(); + + // A third one + e = new TarArchiveEntry(ADDITIONAL_FILE); + e.setSize(13); + tOut.putArchiveEntry(e); + data = "Test String 2".getBytes(); + tOut.write(data, 0, data.length); + tOut.closeArchiveEntry(); + } + + // Create the datasource + var workdirPath = fileSystem.getPath(WORK_DIR); + DataSource dataSource = DataSourceUtil.createDataSource(workdirPath, TAR_FILENAME, null); + + // Assertions on the files in the archive + assertTrue(dataSource.exists(UNRELATED_FILE)); + assertFalse(dataSource.exists("not.tar.gz")); + assertTrue(dataSource.exists(null, MAIN_EXT)); + assertTrue(dataSource.exists(ADDITIONAL_SUFFIX, ADDITIONAL_EXT)); + assertFalse(dataSource.exists("-not", "there")); + try (InputStream is = dataSource.newInputStream(UNRELATED_FILE)) { + assertEquals("Test String", new String(is.readAllBytes())); + } + } + + private OutputStream getCompressedOutputStream(OutputStream os, CompressionFormat compressionFormat) throws IOException { + return switch (compressionFormat) { + case GZIP -> new GzipCompressorOutputStream(os); + case BZIP2 -> new BZip2CompressorOutputStream(os); + case XZ -> new XZCompressorOutputStream(os); + case ZSTD -> new ZstdCompressorOutputStream(os); + default -> os; + }; + } +} diff --git a/commons/src/test/java/com/powsybl/commons/datasource/ZipArchiveDataSourceTest.java b/commons/src/test/java/com/powsybl/commons/datasource/ZipArchiveDataSourceTest.java index f715be6551b..845ea5f26d1 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/ZipArchiveDataSourceTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/ZipArchiveDataSourceTest.java @@ -55,24 +55,16 @@ void testConstructors() { DataSourceObserver observer = new DefaultDataSourceObserver(); // Check constructors - checkDataSource(new ZipArchiveDataSource(testDir, "foo_bar.zip", "foo", "iidm", observer), "foo_bar.zip", "foo", "iidm", observer); - checkDataSource(new ZipArchiveDataSource(testDir, "foo_bar.zip", "foo", "iidm"), "foo_bar.zip", "foo", "iidm", null); - checkDataSource(new ZipArchiveDataSource(testDir, "foo", "iidm", observer), "foo.iidm.zip", "foo", "iidm", observer); - checkDataSource(new ZipArchiveDataSource(testDir, "foo", "", observer), "foo.zip", "foo", "", observer); - checkDataSource(new ZipArchiveDataSource(testDir, "foo", "iidm"), "foo.iidm.zip", "foo", "iidm", null); - checkDataSource(new ZipArchiveDataSource(testDir, "foo", (String) null), "foo.zip", "foo", null, null); - checkDataSource(new ZipArchiveDataSource(testDir, "foo", ""), "foo.zip", "foo", "", null); - checkDataSource(new ZipArchiveDataSource(testDir, "foo", observer), "foo.zip", "foo", null, observer); - checkDataSource(new ZipArchiveDataSource(testDir, "foo"), "foo.zip", "foo", null, null); - checkDataSource(new ZipArchiveDataSource(testDir.resolve("foo_bar.zip")), "foo_bar.zip", "foo_bar", null, null); - } - - private void checkDataSource(ZipArchiveDataSource dataSource, String zipFileName, String baseName, String dataExtension, DataSourceObserver observer) { - assertEquals(testDir, dataSource.getDirectory()); - assertEquals(dataExtension, dataSource.getDataExtension()); - assertEquals(zipFileName, dataSource.getArchiveFilePath().getFileName().toString()); - assertEquals(baseName, dataSource.getBaseName()); - assertEquals(observer, dataSource.getObserver()); + checkDataSource(new ZipArchiveDataSource(testDir, "foo_bar.zip", "foo", "iidm", observer), "foo_bar.zip", "foo", "iidm", archiveFormat, compressionFormat, observer); + checkDataSource(new ZipArchiveDataSource(testDir, "foo_bar.zip", "foo", "iidm"), "foo_bar.zip", "foo", "iidm", archiveFormat, compressionFormat, null); + checkDataSource(new ZipArchiveDataSource(testDir, "foo", "iidm", observer), "foo.iidm.zip", "foo", "iidm", archiveFormat, compressionFormat, observer); + checkDataSource(new ZipArchiveDataSource(testDir, "foo", "", observer), "foo.zip", "foo", "", archiveFormat, compressionFormat, observer); + checkDataSource(new ZipArchiveDataSource(testDir, "foo", "iidm"), "foo.iidm.zip", "foo", "iidm", archiveFormat, compressionFormat, null); + checkDataSource(new ZipArchiveDataSource(testDir, "foo", (String) null), "foo.zip", "foo", null, archiveFormat, compressionFormat, null); + checkDataSource(new ZipArchiveDataSource(testDir, "foo", ""), "foo.zip", "foo", "", archiveFormat, compressionFormat, null); + checkDataSource(new ZipArchiveDataSource(testDir, "foo", observer), "foo.zip", "foo", null, archiveFormat, compressionFormat, observer); + checkDataSource(new ZipArchiveDataSource(testDir, "foo"), "foo.zip", "foo", null, archiveFormat, compressionFormat, null); + checkDataSource(new ZipArchiveDataSource(testDir.resolve("foo_bar.zip")), "foo_bar.zip", "foo_bar", null, archiveFormat, compressionFormat, null); } @Override @@ -161,8 +153,12 @@ void createZipDataSourceWithMoreThanOneDot() throws IOException { out.closeEntry(); } + + // Create the datasource var workdirPath = fileSystem.getPath(WORK_DIR); DataSource dataSource = DataSourceUtil.createDataSource(workdirPath, ZIP_FILENAME, null); + + // Assertions on the files in the archive assertTrue(dataSource.exists(UNRELATED_FILE)); assertFalse(dataSource.exists("not.zip")); assertTrue(dataSource.exists(null, MAIN_EXT)); diff --git a/commons/src/test/resources/foo.iidm.tar.gz b/commons/src/test/resources/foo.iidm.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..fdfd41babfb26c45bc978c1c2bdae2411bdc58fe GIT binary patch literal 230 zcmb2|=3oE==C{`Zxta`Qj(@Z*=kgEu|MK7MLZ#ru1m0!)zFbMP@?!UHjnGfk4?cDO zxocum^T)ekmfO$s*JbAx3QY{~&sJUcclX}=FMjfDncyWXX?7{YF=(QXtJ6lcFRQX| zHTw%ajIG^v{QG_1-v2AM<=iUS{6!~0b82zmZF!&X=NH!N{{CltlI`#L#24~E*N0zz zI(2pTrT35iCTrZ0_;pj!07Q&z6aWAK literal 0 HcmV?d00001 From 493e410fb4446329d60223350e29564df0058e1f Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Wed, 17 Jul 2024 11:47:36 +0200 Subject: [PATCH 02/12] add test Signed-off-by: Nicolas Rol --- .../com/powsybl/commons/datasource/TarArchiveDataSourceTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java b/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java index 1c1a7989b58..eb8e6482396 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java @@ -69,6 +69,7 @@ void testConstructors() { checkDataSource(new TarArchiveDataSource(testDir, "foo", "", compressionFormat), "foo.tar.gz", "foo", "", archiveFormat, compressionFormat, null); checkDataSource(new TarArchiveDataSource(testDir, "foo", compressionFormat, observer), "foo.tar.gz", "foo", null, archiveFormat, compressionFormat, observer); checkDataSource(new TarArchiveDataSource(testDir, "foo", compressionFormat), "foo.tar.gz", "foo", null, archiveFormat, compressionFormat, null); + checkDataSource(new TarArchiveDataSource(testDir, "foo"), "foo.tar", "foo", null, archiveFormat, null, null); checkDataSource(new TarArchiveDataSource(testDir.resolve("foo_bar.tar.gz")), "foo_bar.tar.gz", "foo_bar", "", archiveFormat, compressionFormat, null); } From c7f4c46eff2c97125ece45adfba2b8c6999ceb44 Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Wed, 17 Jul 2024 15:18:53 +0200 Subject: [PATCH 03/12] fix tests Signed-off-by: Nicolas Rol --- .../datasource/TarArchiveDataSource.java | 22 +++++++++---------- .../AbstractArchiveDataSourceTest.java | 16 ++++++++++++++ .../datasource/TarArchiveDataSourceTest.java | 11 ++++++++-- .../datasource/ZipArchiveDataSourceTest.java | 5 +++++ 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java index ccce9e52997..295574f1a39 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java @@ -156,7 +156,9 @@ public InputStream newInputStream(String suffix, String ext) throws IOException public InputStream newInputStream(String fileName) throws IOException { Objects.requireNonNull(fileName); Path tarFilePath = getArchiveFilePath(); - try { + + // If the file is in the archive, we can open it + if (entryExists(tarFilePath, fileName)) { InputStream fis = Files.newInputStream(tarFilePath); BufferedInputStream bis = new BufferedInputStream(fis); InputStream is = getCompressedInputStream(bis, this.compressionFormat); @@ -167,10 +169,8 @@ public InputStream newInputStream(String fileName) throws IOException { return observer != null ? new ObservableInputStream(tais, tarFilePath + ":" + fileName, observer) : tais; } } - return null; - } catch (IOException e) { - return null; } + return null; } private static final class TarEntryOutputStream extends ForwardingOutputStream { @@ -220,13 +220,13 @@ private void compressTarFile() throws IOException { } } - private static OutputStream getCompressedOutputStream(OutputStream is, CompressionFormat compressionFormat) throws IOException { - return switch (compressionFormat) { - case GZIP -> new GzipCompressorOutputStream(is); - case BZIP2 -> new BZip2CompressorOutputStream(is); - case XZ -> new XZCompressorOutputStream(is); - case ZSTD -> new ZstdCompressorOutputStream(is); - default -> is; + private static OutputStream getCompressedOutputStream(OutputStream os, CompressionFormat compressionFormat) throws IOException { + return compressionFormat == null ? os : switch (compressionFormat) { + case GZIP -> new GzipCompressorOutputStream(os); + case BZIP2 -> new BZip2CompressorOutputStream(os); + case XZ -> new XZCompressorOutputStream(os); + case ZSTD -> new ZstdCompressorOutputStream(os); + default -> os; }; } diff --git a/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java b/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java index cb04da1c71b..17b5beda764 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java @@ -93,4 +93,20 @@ void testErrorOnAppend() throws IOException { }); assertEquals(appendException, exception.getMessage()); } + + protected abstract AbstractArchiveDataSource createArchiveDataSource(); + + @Test + void testMissingArchive() throws IOException { + AbstractArchiveDataSource dataSource = createArchiveDataSource(); + assertFalse(dataSource.exists("test.bar")); + assertNull(dataSource.newInputStream("test.bar")); + } + + @Test + void testWrongTypeOfFile() throws IOException { + Files.createFile(testDir.resolve("foo.bar")); + AbstractArchiveDataSource dataSource = createArchiveDataSource(); + assertFalse(dataSource.exists("test.bar")); + } } diff --git a/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java b/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java index eb8e6482396..fb2c51965a9 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java @@ -89,11 +89,18 @@ protected DataSource createDataSource(DataSourceObserver observer) { return new TarArchiveDataSource(testDir, "foo", "iidm", compressionFormat, observer); } + @Override + protected AbstractArchiveDataSource createArchiveDataSource() { + return new TarArchiveDataSource(testDir, "foo.bar", "foo", null, compressionFormat, null); + } + static Stream provideArgumentsForWriteThenReadTest() { return Stream.of( Arguments.of("foo", "iidm", CompressionFormat.GZIP), Arguments.of("foo", "", CompressionFormat.XZ), - Arguments.of("foo", "v3", CompressionFormat.ZSTD) + Arguments.of("foo", "v3", CompressionFormat.ZSTD), + Arguments.of("foo", "v3", CompressionFormat.BZIP2), + Arguments.of("foo", "v3", null) ); } @@ -195,7 +202,7 @@ void testTarDataSourceWithMoreThanOneDot() throws IOException { } private OutputStream getCompressedOutputStream(OutputStream os, CompressionFormat compressionFormat) throws IOException { - return switch (compressionFormat) { + return compressionFormat == null ? os : switch (compressionFormat) { case GZIP -> new GzipCompressorOutputStream(os); case BZIP2 -> new BZip2CompressorOutputStream(os); case XZ -> new XZCompressorOutputStream(os); diff --git a/commons/src/test/java/com/powsybl/commons/datasource/ZipArchiveDataSourceTest.java b/commons/src/test/java/com/powsybl/commons/datasource/ZipArchiveDataSourceTest.java index 845ea5f26d1..582f541c04f 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/ZipArchiveDataSourceTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/ZipArchiveDataSourceTest.java @@ -82,6 +82,11 @@ protected DataSource createDataSource(DataSourceObserver observer) { return new ZipArchiveDataSource(testDir, "foo", "iidm", observer); } + @Override + protected AbstractArchiveDataSource createArchiveDataSource() { + return new ZipArchiveDataSource(testDir, "foo.bar", "foo", null, null); + } + static Stream provideArgumentsForWriteThenReadTest() { return Stream.of( Arguments.of("foo", "iidm", CompressionFormat.ZIP), From ec0af62eb25c3262b78ddb707ab9bef64eeffb82 Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Thu, 18 Jul 2024 09:06:29 +0200 Subject: [PATCH 04/12] try to fix issue Signed-off-by: Nicolas Rol --- .../datasource/TarArchiveDataSource.java | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java index 295574f1a39..578775c5fec 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java @@ -7,6 +7,7 @@ */ package com.powsybl.commons.datasource; +import com.powsybl.commons.io.ForwardingInputStream; import com.powsybl.commons.io.ForwardingOutputStream; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; @@ -159,18 +160,31 @@ public InputStream newInputStream(String fileName) throws IOException { // If the file is in the archive, we can open it if (entryExists(tarFilePath, fileName)) { - InputStream fis = Files.newInputStream(tarFilePath); - BufferedInputStream bis = new BufferedInputStream(fis); - InputStream is = getCompressedInputStream(bis, this.compressionFormat); - TarArchiveInputStream tais = new TarArchiveInputStream(is); + InputStream is = new TarEntryInputStream(tarFilePath, fileName, compressionFormat); + return observer != null ? new ObservableInputStream(is, tarFilePath + ":" + fileName, observer) : is; + } + return null; + } + + private static final class TarEntryInputStream extends ForwardingInputStream { + + private TarEntryInputStream(Path tarFilePath, String fileName, CompressionFormat compressionFormat) throws IOException { + super(setStreamToFile(getTmpStream(tarFilePath, compressionFormat), fileName)); + } + + private static TarArchiveInputStream getTmpStream(Path tarFilePath, CompressionFormat compressionFormat) throws IOException { + return new TarArchiveInputStream(getCompressedInputStream(new BufferedInputStream(Files.newInputStream(tarFilePath)), compressionFormat)); + } + + private static InputStream setStreamToFile(TarArchiveInputStream tais, String fileName) throws IOException { TarArchiveEntry entry; while ((entry = tais.getNextEntry()) != null) { if (entry.getName().equals(fileName)) { - return observer != null ? new ObservableInputStream(tais, tarFilePath + ":" + fileName, observer) : tais; + return tais; } } + return null; } - return null; } private static final class TarEntryOutputStream extends ForwardingOutputStream { From 19e509b4db3307fc99dd7d6a2148b81888ec48ee Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Thu, 18 Jul 2024 10:35:26 +0200 Subject: [PATCH 05/12] add coverage Signed-off-by: Nicolas Rol --- .../AbstractArchiveDataSourceTest.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java b/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java index 17b5beda764..2fb92472f35 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java @@ -15,6 +15,7 @@ import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; @@ -77,6 +78,27 @@ void testFileInSubfolder() throws IOException { assertTrue(files.contains("subfolder/foo_baz.iidm")); } + @Test + void testStreams() throws IOException { + // Create file + createArchiveAndFiles(archiveWithSubfolders); + Path path = fileSystem.getPath(archiveWithSubfolders); + + // Datasource with an observer + DataSourceObserver observer = new DefaultDataSourceObserver(); + DataSource dataSource = DataSourceUtil.createDataSource(fileSystem.getPath("."), archiveWithSubfolders, observer); + assertInstanceOf(ObservableInputStream.class, dataSource.newInputStream("foo.iidm")); + assertInstanceOf(ObservableOutputStream.class, dataSource.newOutputStream("test.iidm", false)); + + // Datasource without an observer + dataSource = DataSource.fromPath(path); + try (InputStream inputStream = dataSource.newInputStream("foo.iidm"); + OutputStream outputStream = dataSource.newOutputStream("test.iidm", false)) { + assertFalse(inputStream instanceof ObservableInputStream); + assertFalse(outputStream instanceof ObservableOutputStream); + } + } + @Test void testErrorOnAppend() throws IOException { // File From 8c7466269de5c64d742d4e6b8bd3d274702c2804 Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Thu, 18 Jul 2024 10:39:26 +0200 Subject: [PATCH 06/12] use private constructor to avoid multiple FileInformation Signed-off-by: Nicolas Rol --- .../powsybl/commons/datasource/TarArchiveDataSource.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java index 578775c5fec..8a817b2a8b9 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java @@ -76,9 +76,11 @@ public TarArchiveDataSource(Path directory, String baseName) { } public TarArchiveDataSource(Path tarFile) { - this(tarFile.getParent(), (new FileInformation(tarFile.getFileName().toString(), false)).getBaseName(), - (new FileInformation(tarFile.getFileName().toString(), false)).getDataExtension(), - (new FileInformation(tarFile.getFileName().toString(), false)).getCompressionFormat()); + this(tarFile.getParent(), new FileInformation(tarFile.getFileName().toString(), false)); + } + + private TarArchiveDataSource(Path directory, FileInformation fileInformation) { + this(directory, fileInformation.getBaseName(), fileInformation.getDataExtension(), fileInformation.getCompressionFormat()); } /** From 03312259a8e67049b6dbaa8eca3935cea362f745 Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Thu, 18 Jul 2024 10:45:15 +0200 Subject: [PATCH 07/12] make tar work like zip in listFiles Signed-off-by: Nicolas Rol --- .../com/powsybl/commons/datasource/TarArchiveDataSource.java | 3 --- .../com/powsybl/commons/datasource/ZipArchiveDataSource.java | 1 - 2 files changed, 4 deletions(-) diff --git a/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java index 8a817b2a8b9..502386ba220 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java @@ -101,13 +101,10 @@ public Set listNames(String regex) throws IOException { TarArchiveInputStream tar = new TarArchiveInputStream(cis)) { ArchiveEntry entry; while ((entry = tar.getNextEntry()) != null) { - // Consider only files located at the root of the archive if (!entry.isDirectory() - && entry.getName().indexOf('/') == entry.getName().lastIndexOf('/') && p.matcher(entry.getName()).matches()) { names.add(entry.getName()); } - } } return names; diff --git a/commons/src/main/java/com/powsybl/commons/datasource/ZipArchiveDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/ZipArchiveDataSource.java index 8a63508c040..8372a0fe63e 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/ZipArchiveDataSource.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/ZipArchiveDataSource.java @@ -186,7 +186,6 @@ public OutputStream newOutputStream(String suffix, String ext, boolean append) t @Override public Set listNames(String regex) throws IOException { - // Consider only files in the given folder, do not go into folders Pattern p = Pattern.compile(regex); Set names = new HashSet<>(); Path zipFilePath = getArchiveFilePath(); From 2a4ba2706fa37283012f5848522cbad07aa6516b Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Fri, 19 Jul 2024 10:21:21 +0200 Subject: [PATCH 08/12] change test to show that files in every subfolder is reached Signed-off-by: Nicolas Rol --- .../AbstractArchiveDataSourceTest.java | 3 ++- commons/src/test/resources/foo.iidm.tar.gz | Bin 230 -> 276 bytes commons/src/test/resources/foo.iidm.zip | Bin 654 -> 1029 bytes 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java b/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java index 2fb92472f35..f8f51372932 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java @@ -71,11 +71,12 @@ void testFileInSubfolder() throws IOException { // All the files are listed, no filter is applied Set files = dataSource.listNames(".*"); - assertEquals(3, files.size()); + assertEquals(4, files.size()); assertTrue(files.contains("foo.iidm")); assertTrue(files.contains("foo_bar.iidm")); assertFalse(files.contains("foo_baz.iidm")); assertTrue(files.contains("subfolder/foo_baz.iidm")); + assertTrue(files.contains("subfolder/subsubfolder/foo.v3.iidm")); } @Test diff --git a/commons/src/test/resources/foo.iidm.tar.gz b/commons/src/test/resources/foo.iidm.tar.gz index fdfd41babfb26c45bc978c1c2bdae2411bdc58fe..12f886ea42b160f5f3e7261381b19a5a8fb35380 100644 GIT binary patch literal 276 zcmV+v0qg!BiwFP!000001MSvbs=_c31>ij2Rdfa3nY5EENBk=k)IeK(@aZn*F2#j8 zK_pN9fvITE*M}BbXrMi3Qim{Un&rl1M9R6!{ONUb{HMz~5yO=u^5WUCAud_GDJbWn zE_Pi%Ym36PVZCh@*Ppw0|3mewuK#)WBH(T65T^54o3#V(=ijCLd#CxYh%NFzgqVMe z0vN4sH?PBbxoUs&y(1>vd%2N+iuu>W>Vp*LzpyifoZ^8IxfO!noAX2H|_ufKtA1JM5i aK>q^(0000000000e5VsU6=GfhC;$Mi)QdF$ literal 230 zcmb2|=3oE==C{`Zxta`Qj(@Z*=kgEu|MK7MLZ#ru1m0!)zFbMP@?!UHjnGfk4?cDO zxocum^T)ekmfO$s*JbAx3QY{~&sJUcclX}=FMjfDncyWXX?7{YF=(QXtJ6lcFRQX| zHTw%ajIG^v{QG_1-v2AM<=iUS{6!~0b82zmZF!&X=NH!N{{CltlI`#L#24~E*N0zz zI(2pTrT35iCTrZ0_;pj!07Q&z6aWAK diff --git a/commons/src/test/resources/foo.iidm.zip b/commons/src/test/resources/foo.iidm.zip index 37c5709db71d0266eb19fff06e8bbb197e0c96fa..acc26ee76a4ad694b63619980a07aa701fffe39e 100644 GIT binary patch literal 1029 zcmWIWW@h1H0D;AR)e*-`cUrRo*&xipAj6QBpRbphnUWhC!pXo~c#k=)SYcLLX$3a} zBa6Y3Bbi{LB(=DND*$fHN}w?}QcA+vfJT5Y55$=Gq{JeGQFXYDGD0$ITj1vipb>Du z1vIO;G$}1VCndE=AM7X#po45N9ffdDdqpjZx5QA)0D2JUvMPkj;!d%pxnr7#=E);y z4iiT)6zD`WR|Wu0#1eG4U8#iIKv0~N8Kb%wi`h^YGcw6B0^r3<3;)9VY`#L=IG-d7!ArV;*M2 zBfIcAig~EX0BANS8Q?J+Gyaj?ssjvCgj>;41JH0#YQSSSy2F(ikR4u&+i;YWfy)!D UpiBwOnQRP#ObiV7K(WsN0Bkz@Hvj+t delta 153 zcmZqW=wsy#@MdP=VgLcA>e`756xf;1v!$_|XPcbMs5SAUGIQK1w#nj*@)NmKng7m& zFatF~%!#!Uyo^k8%y7+(Pi>aCfmuc j808pePUd9tWqdF>oJmuJl?|kY1qgi@85necn1KNRdy6At From 29a1acb152bc1b9246f70a72983f321c06a1d6ce Mon Sep 17 00:00:00 2001 From: Florian Dupuy Date: Thu, 1 Aug 2024 17:07:15 +0200 Subject: [PATCH 09/12] Fix after unproper rebase Signed-off-by: Florian Dupuy --- .../datasource/AbstractArchiveDataSourceTest.java | 2 +- .../commons/datasource/TarArchiveDataSourceTest.java | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java b/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java index f8f51372932..872e93a9634 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/AbstractArchiveDataSourceTest.java @@ -82,7 +82,7 @@ void testFileInSubfolder() throws IOException { @Test void testStreams() throws IOException { // Create file - createArchiveAndFiles(archiveWithSubfolders); + createFiles(archiveWithSubfolders); Path path = fileSystem.getPath(archiveWithSubfolders); // Datasource with an observer diff --git a/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java b/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java index fb2c51965a9..6c4ebca3f22 100644 --- a/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java +++ b/commons/src/test/java/com/powsybl/commons/datasource/TarArchiveDataSourceTest.java @@ -53,6 +53,14 @@ void setUp() throws Exception { compressionFormat = CompressionFormat.GZIP; } + @Override + protected String getFileName(String baseName, String dataExtension, CompressionFormat compressionFormat) { + return testDir + "/" + baseName + + (dataExtension == null || dataExtension.isEmpty() ? "" : "." + dataExtension) + + (archiveFormat == null ? "" : "." + archiveFormat.getExtension()) + + (compressionFormat == null ? "" : "." + compressionFormat.getExtension()); + } + @Test @Override void testConstructors() { @@ -119,7 +127,7 @@ static Stream provideArgumentsForClassAndListingTest() { } @Override - protected void createArchiveAndFiles(String fileName) throws IOException { + protected void createFiles(String fileName) throws IOException { // File information FileInformation fileInformation = new FileInformation(fileName); From bc0a5ab1ebbbdb6472dc93db1d3da498e1a7e302 Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Thu, 1 Aug 2024 16:53:24 +0200 Subject: [PATCH 10/12] separate if cases Signed-off-by: Nicolas Rol --- .../com/powsybl/commons/datasource/FileInformation.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/commons/src/main/java/com/powsybl/commons/datasource/FileInformation.java b/commons/src/main/java/com/powsybl/commons/datasource/FileInformation.java index 2de0d0bae75..15bad57ac66 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/FileInformation.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/FileInformation.java @@ -64,9 +64,12 @@ private void computeInformation(String fileName, boolean dataSourceInitializatio if (compressionFormat == CompressionFormat.ZIP) { archiveFormat = ArchiveFormat.ZIP; fileNameWithoutCompressionNorArchive = fileNameWithoutCompressionExtension; + } else if (ArchiveFormat.TAR.getExtension().equals(fileNameWithoutCompressionExtension.substring(currentDotIndex + 1))) { + archiveFormat = ArchiveFormat.TAR; + fileNameWithoutCompressionNorArchive = fileNameWithoutCompressionExtension.substring(0, currentDotIndex); } else { - archiveFormat = "tar".equals(fileNameWithoutCompressionExtension.substring(currentDotIndex + 1)) ? ArchiveFormat.TAR : null; - fileNameWithoutCompressionNorArchive = archiveFormat == null ? fileNameWithoutCompressionExtension : fileNameWithoutCompressionExtension.substring(0, currentDotIndex); + archiveFormat = null; + fileNameWithoutCompressionNorArchive = fileNameWithoutCompressionExtension; } // Last dot index From 9c123b2642d57b9ee5011a857e3eab75e7afb770 Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Thu, 1 Aug 2024 17:37:40 +0200 Subject: [PATCH 11/12] use Files.size Signed-off-by: Nicolas Rol --- .../commons/datasource/TarArchiveDataSource.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java index 502386ba220..534992a3a9e 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java @@ -257,21 +257,21 @@ public void close() throws IOException { taos.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); taos.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); - // Copy content of temporary stream file into an entry of the temporary archive - try (InputStream is = Files.newInputStream(getTmpStreamFilePath(tarFilePath))) { + // Temporary stream file path + Path tmpStreamFilePath = getTmpStreamFilePath(tarFilePath); - // Content of the stream - byte[] streamContent = is.readAllBytes(); + // Copy content of temporary stream file into an entry of the temporary archive + try (InputStream is = Files.newInputStream(tmpStreamFilePath)) { // New tar entry TarArchiveEntry entry = new TarArchiveEntry(fileName); - entry.setSize(streamContent.length); + entry.setSize(Files.size(tmpStreamFilePath)); // New file to add taos.putArchiveEntry(entry); // Write the data in the entry - taos.write(streamContent); + taos.write(is.readAllBytes()); // close new entry taos.closeArchiveEntry(); From 693b4d533a941a89f693e29afdf5f29b00ed96b6 Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Fri, 2 Aug 2024 15:48:04 +0200 Subject: [PATCH 12/12] use of ByteStreams.copy Signed-off-by: Nicolas Rol --- .../com/powsybl/commons/datasource/TarArchiveDataSource.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java index 534992a3a9e..c79f7872f72 100644 --- a/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java +++ b/commons/src/main/java/com/powsybl/commons/datasource/TarArchiveDataSource.java @@ -7,6 +7,7 @@ */ package com.powsybl.commons.datasource; +import com.google.common.io.ByteStreams; import com.powsybl.commons.io.ForwardingInputStream; import com.powsybl.commons.io.ForwardingOutputStream; import org.apache.commons.compress.archivers.ArchiveEntry; @@ -271,7 +272,7 @@ public void close() throws IOException { taos.putArchiveEntry(entry); // Write the data in the entry - taos.write(is.readAllBytes()); + ByteStreams.copy(is, taos); // close new entry taos.closeArchiveEntry();