Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: add support for gzip archive files #157

Merged
merged 6 commits into from
Jan 17, 2023
Merged

feat: add support for gzip archive files #157

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1082b29
feat: add support for tgz archive files
Jan 13, 2023
513e5fb
feat: add support for tgz archive files - test on linux as well as wi…
Jan 13, 2023
c67b586
feat: add support for tgz archive files - apply PR review comments
Jan 13, 2023
6e04d6f
feat: add support for tgz archive files - fix issue in endsWith
Jan 13, 2023
d567d8f
Merge branch 'sevdokimov:master' into master
mhewedy Jan 16, 2023
1dfcfd3
feat: add support for gzip archive files - polish
Jan 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions log-viewer-cli/bin/config.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ log-viewer.use-web-socket=true

log-viewer.backdoor_server.port=9595

log-viewer.unpack-archive=false

// List of file patterns describing visible files and subdirectories
logs = [
{
Expand Down
2 changes: 1 addition & 1 deletion log-viewer-frontend/src/app/log-navigator/log-navigator.component.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ export class LogNavigatorComponent implements OnInit, AfterViewInit {
} else {
let fileType = fsItem.type;

if (fileType === 'log' || fileType === 'out' || fileType === 'text') {
if (fileType === 'log' || fileType === 'out' || fileType === 'text' || fileType === 'gz') {
this.openFile.emit({path: fsItem.path, isCtrlClick: inNewWindow});
}
}
Expand Down
56 changes: 39 additions & 17 deletions log-viewer/src/main/java/com/logviewer/data2/Log.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import org.springframework.lang.Nullable;

import java.io.EOFException;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.SeekableByteChannel;
Expand All @@ -30,6 +32,9 @@
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.zip.CRC32;
import java.util.zip.GZIPInputStream;

import static com.logviewer.files.FileTypes.GZ;

public class Log implements LogView {

Expand All @@ -56,7 +61,7 @@ public class Log implements LogView {

private final Object logChangedTaskKey = new Object();

private final Path file;
private Path file;

private final String id;

Expand All @@ -77,6 +82,8 @@ public class Log implements LogView {
private LvFileAccessManager accessManager;
@Value("${log-viewer.parser.max-unparsable-block-size:2097152}") // 2Mb
private long unparsableBlockMaxSize;
@Value("${log-viewer.unpack-archive:false}")
private boolean unpackArchive;

private final MultiListener<Consumer<FileAttributes>> changeListener = new MultiListener<>(this::createFileListener);

Expand Down Expand Up @@ -141,6 +148,25 @@ public Snapshot createSnapshot() {
}
}

private void decompressAndCopyGZipFile() throws IOException {

File tempFile = File.createTempFile("log-viewer-", "-" + file.getName(file.getNameCount() - 1) + ".tmp");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suggest to change it to NIO2

Path tempFile = Files.createTempFile("log-viewer-", "-" + file.getName(file.getNameCount() - 1) + ".tmp");
Files.deleteIfExists(tempFile);

Copy link
Author

@mhewedy mhewedy Jan 16, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did some searches, and find out there's no direct way in java nio API to get from gz to SeekableByteChannel.
I have to introduce some external dependencies and/or write custom code to accomplish this.

I appreciate your help on this.

tempFile.deleteOnExit();

try (GZIPInputStream gis = new GZIPInputStream(
Files.newInputStream(file.toFile().toPath()));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it really necessary to convert it to File, and then back to Path? Or is it just mistake?

FileOutputStream fos = new FileOutputStream(tempFile)) {

byte[] buffer = new byte[1024 * 10];
int len;
while ((len = gis.read(buffer)) > 0) {
fos.write(buffer, 0, len);
}
}

this.file = tempFile.toPath();
}

public class LogSnapshot implements Snapshot {

private final long size;
Expand Down Expand Up @@ -183,8 +209,7 @@ public class LogSnapshot implements Snapshot {

if (cachedHashTimestamp == lastModification) {
hash = cachedHash;
}
else {
} else {
hash = calculateHash(size);

if (!hash.equals(cachedHash)) {
Expand All @@ -200,8 +225,7 @@ public class LogSnapshot implements Snapshot {
success = true;
} catch (IOException e) {
error = e;
}
finally {
} finally {
if (!success)
Utils.closeQuietly(this);
}
Expand All @@ -228,6 +252,9 @@ private SeekableByteChannel getChannel() throws IOException {
if (error != null)
throw new IOException(error);

if (unpackArchive && GZ.getPattern().matcher(file.toString()).matches())
decompressAndCopyGZipFile();

channel = Files.newByteChannel(file, StandardOpenOption.READ);
}

Expand Down Expand Up @@ -317,8 +344,7 @@ public boolean processRecordsBack(long position, boolean fromPrevLine, Predicate
if (fromPrevLine) {
if (!buf.loadPrevLine(firstLine, position))
return true;
}
else {
} else {
buf.loadLine(firstLine, position);
}

Expand All @@ -341,8 +367,7 @@ public boolean processRecordsBack(long position, boolean fromPrevLine, Predicate

if (reader.canAppendTail()) {
appendTail(buf, reader, line.getEnd(), unparsedEnd);
}
else {
} else {
if (!consumer.test(createUnparsedRecord(buf, unparsedStart, unparsedEnd)))
return false;
}
Expand Down Expand Up @@ -378,8 +403,7 @@ public boolean processRecordsBack(long position, boolean fromPrevLine, Predicate

if (reader.canAppendTail()) {
appendTail(buf, reader, line.getEnd(), unparsedEnd);
}
else {
} else {
if (!consumer.test(createUnparsedRecord(buf, unparsedStart, unparsedEnd)))
return false;
}
Expand Down Expand Up @@ -414,8 +438,7 @@ public boolean processRecords(long position, boolean fromNextLine, Predicate<Log
if (fromNextLine) {
if (!buf.loadNextLine(line, position))
return true;
}
else {
} else {
buf.loadLine(line, position);
}

Expand Down Expand Up @@ -529,8 +552,7 @@ public boolean processRecords(long position, boolean fromNextLine, Predicate<Log
if (!consumer.test(createUnparsedRecord(buf, unparsedStart, prevEnd)))
return false;
}
}
else {
} else {
if (!consumer.test(reader.buildRecord().setLogId(id)))
return false;
}
Expand Down Expand Up @@ -603,7 +625,7 @@ private String calculateHash(long fileSize) throws LogCrashedException, IOExcept
crc.update(buf.array());

int hash = (int) crc.getValue();
long hashWithLength = (hash & 0xffff_ffffL) | ((long)hashSize << 32);
long hashWithLength = (hash & 0xffff_ffffL) | ((long) hashSize << 32);

return Long.toHexString(hashWithLength);
} catch (EOFException e) {
Expand All @@ -621,7 +643,7 @@ public boolean isValidHash(@NonNull String hash) {

int hashSize = (int) ((tLong >>> 32) & 0xff);

if (hashSize == hashSize(size) ) { // Compare size of block used to has calculation.
if (hashSize == hashSize(size)) { // Compare size of block used to has calculation.
return hash.equals(this.hash);
}

Expand Down