Skip to content
This repository has been archived by the owner on Sep 1, 2022. It is now read-only.

Fixes #445: GRIB1 data length overflow #1005

Merged
merged 1 commit into from
Jan 16, 2018
Merged

Fixes #445: GRIB1 data length overflow #1005

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 44 additions & 4 deletions grib/src/main/java/ucar/nc2/grib/grib1/Grib1RecordScanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,14 @@ static public boolean isValidFile(RandomAccessFile raf) {
int len = GribNumbers.uint3(raf);
int edition = raf.read(); // read at byte 8
if (edition != 1) return false;


/* Due to a trick done by ECMWF's GRIBEX to support large GRIBs, we need a special treatment
* to fix the length of the GRIB message. See:
* https://software.ecmwf.int/wiki/display/EMOS/Changes+in+cycle+000281
* https://github.com/Unidata/thredds/issues/445
*/
len = getFixedTotalLengthEcmwfLargeGrib(raf,len);

// check ending = 7777
if (len > raf.length()) return false;
if (allowBadIsLength) return true;
Expand All @@ -100,7 +107,40 @@ static public boolean isValidFile(RandomAccessFile raf) {
}
}

////////////////////////////////////////////////////////////
static int getFixedTotalLengthEcmwfLargeGrib(RandomAccessFile raf, int len) throws IOException{
int lenActual=len;
//int lenS4Actual=0;
//int sizeSection4=0;
if ((len & 0x800000) == 0x800000) {
long pos0 = raf.getFilePointer(); // remember the actual pos
int lenS1 = GribNumbers.uint3(raf); // section1Length
raf.skipBytes(1); // table2Version
if (GribNumbers.uint(raf) == 98) { // center (if ECMWF make the black magic)
raf.skipBytes(2); // generatingProcessIdentifier, gridDefinition
int s1f = GribNumbers.uint(raf); // section1Flags
raf.skipBytes(lenS1 - (3 + 5)); // skips to next section
int lenS2 = 0;
int lenS3 = 0;
if ((s1f & 128) == 128) { // section2 GDS exists
lenS2 = GribNumbers.uint3(raf); // section2Length
raf.skipBytes(lenS2 - 3); // skips to next section
}
if ((s1f & 64) == 64) { // section3 BMS exists
lenS3 = GribNumbers.uint3(raf); // section3Length
raf.skipBytes(lenS3 - 3); // skips to next section
}
int lenS4 = GribNumbers.uint3(raf); // section4Length
if (lenS4 < 120) { // here we are!!!!
lenActual = (len & 0x7FFFFF) * 120 - lenS4 + 4; // the actual totalLength
//lenS4Actual = lenActual - 8 - lenS1 - lenS2 - lenS3 - 4; // the actual length for section4
}
}
raf.seek(pos0); // recall the pos
}
return lenActual;
}

////////////////////////////////////////////////////////////

private Map<Long, Grib1SectionGridDefinition> gdsMap = new HashMap<>();
private ucar.unidata.io.RandomAccessFile raf = null;
Expand Down Expand Up @@ -199,10 +239,10 @@ public Grib1Record next() throws IOException {
if (debug) System.out.printf(" read until %d grib ending at %d header ='%s' foundEnding=%s%n",
raf.getFilePointer(), ending, StringUtil2.cleanup(header), foundEnding);

if (!foundEnding && allowBadIsLength)
if (!foundEnding && (allowBadIsLength || is.isMessageLengthFixed))
foundEnding = checkEnding(dataSection.getStartingPosition() + dataSection.getLength());

if (!foundEnding && allowBadDsLength) {
if (!foundEnding && (allowBadDsLength || is.isMessageLengthFixed)) {
foundEnding = true;
}

Expand Down
10 changes: 9 additions & 1 deletion grib/src/main/java/ucar/nc2/grib/grib1/Grib1SectionIndicator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ public class Grib1SectionIndicator {

private final long messageLength;
private final long startPos;
private final int messageLengthNotFixed;
boolean isMessageLengthFixed=false;

/**
* Read Grib2SectionIndicator from raf.
Expand All @@ -65,15 +67,21 @@ public Grib1SectionIndicator(RandomAccessFile raf) throws IOException {
if (b[i] != MAGIC[i])
throw new IllegalArgumentException("Not a GRIB record");

messageLength = GribNumbers.uint3(raf);
messageLengthNotFixed = GribNumbers.uint3(raf);
int edition = raf.read();
if (edition != 1)
throw new IllegalArgumentException("Not a GRIB-1 record");
messageLength = Grib1RecordScanner.getFixedTotalLengthEcmwfLargeGrib(raf, messageLengthNotFixed);
if(messageLength!=messageLengthNotFixed) {
isMessageLengthFixed = true;
}
}

public Grib1SectionIndicator(long startPos, long messageLength) {
this.startPos = startPos;
this.messageLength = messageLength;
this.messageLengthNotFixed = (int)messageLength;
this.isMessageLengthFixed = false;
}

/**
Expand Down