Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

1 12 HDFFV-10865 - merge from dev, HDFArray perf fix #233

Merged
merged 22 commits into from
Dec 29, 2020
Merged

1 12 HDFFV-10865 - merge from dev, HDFArray perf fix #233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
f2bee7d
merge updates from develop
byrnHDF Nov 19, 2020
6ab5469
remove extension on command
byrnHDF Nov 19, 2020
6f38edb
Run autogen before chkmanifest
byrnHDF Nov 20, 2020
ba6f5e2
Merge branch 'hdf5_1_12' of https://github.com/HDFGroup/hdf5.git into…
byrnHDF Nov 20, 2020
31fb51c
Update release text files and whitespace cleanup
byrnHDF Nov 20, 2020
4de97e6
Update list
byrnHDF Nov 20, 2020
5c6e1e9
Merge branch 'hdf5_1_12' of https://github.com/HDFGroup/hdf5.git into…
byrnHDF Nov 23, 2020
4bda2aa
Merge branch 'hdf5_1_12' of https://github.com/HDFGroup/hdf5.git into…
byrnHDF Dec 10, 2020
dcb2c6e
Merge branch 'hdf5_1_12' of https://github.com/HDFGroup/hdf5.git into…
byrnHDF Dec 11, 2020
e37997b
OESS-98 Update plugin build option
byrnHDF Dec 14, 2020
cbf1ec2
OESS-98 fix tools test for plugins
byrnHDF Dec 15, 2020
0c87896
Merge branch 'hdf5_1_12' of https://github.com/HDFGroup/hdf5.git
byrnHDF Dec 15, 2020
1d09369
Fix conflict
byrnHDF Dec 15, 2020
d824f22
Merge branch 'hdf5_1_12' of https://github.com/HDFGroup/hdf5.git into…
byrnHDF Dec 16, 2020
e898add
OESS-98 fix hdf5 link target
byrnHDF Dec 16, 2020
3fd08b4
Merge branch 'hdf5_1_12' of https://github.com/HDFGroup/hdf5.git into…
byrnHDF Dec 17, 2020
1372b71
Merge branch 'hdf5_1_12' of https://github.com/HDFGroup/hdf5.git into…
byrnHDF Dec 22, 2020
450e7d8
Update pkgconfig settings with version - #218
byrnHDF Dec 22, 2020
1c230c3
Merge branch 'hdf5_1_12' of https://github.com/HDFGroup/hdf5.git into…
byrnHDF Dec 23, 2020
7fc79bd
Merge branch 'hdf5_1_12' of https://github.com/HDFGroup/hdf5.git into…
byrnHDF Dec 29, 2020
ae72f7b
HDFFV-10865 - merge from dev, HDFArray perf fix.
byrnHDF Dec 29, 2020
be00691
HDFFV-10865 - fix date
byrnHDF Dec 29, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
304 changes: 120 additions & 184 deletions java/src/hdf/hdf5lib/HDFArray.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

import hdf.hdf5lib.exceptions.HDF5Exception;
import hdf.hdf5lib.exceptions.HDF5JavaException;
import java.util.Arrays;

/**
* This is a class for handling multidimensional arrays for HDF.
Expand Down Expand Up @@ -394,6 +395,7 @@ public Object arrayify(byte[] bytes) throws HDF5JavaException {
throw (ex);
}
_barray = bytes; /* hope that the bytes are correct.... */

if (ArrayDescriptor.dims == 1) {
/* special case */
/* 2 data copies here! */
Expand Down Expand Up @@ -496,8 +498,62 @@ else if (ArrayDescriptor.className.equals("java.lang.Long")) {

Object oo = _theArray;
int n = 0; /* the current byte */
int m = 0; /* the current array index */
int index = 0;
int i;
Object flattenedArray = null;
switch (ArrayDescriptor.NT) {
case 'J':
flattenedArray = (Object) HDFNativeData.byteToLong(_barray);
break;
case 'S':
flattenedArray = (Object) HDFNativeData.byteToShort(_barray);
break;
case 'I':
flattenedArray = (Object) HDFNativeData.byteToInt(_barray);
break;
case 'F':
flattenedArray = (Object) HDFNativeData.byteToFloat(_barray);
break;
case 'D':
flattenedArray = (Object) HDFNativeData.byteToDouble(_barray);
break;
case 'B':
flattenedArray = (Object) _barray;
break;
case 'L':
switch (ArrayDescriptor.className) {
case "java.lang.Byte":
flattenedArray = (Object) ByteToByteObj(_barray);
break;
case "java.lang.Short":
flattenedArray = (Object) ByteToShort(_barray);
break;
case "java.lang.Integer":
flattenedArray = (Object) ByteToInteger(_barray);
break;
case "java.lang.Long":
flattenedArray = (Object) ByteToLongObj(_barray);
break;
case "java.lang.Float":
flattenedArray = (Object) ByteToFloatObj(_barray);
break;
case "java.lang.Double":
flattenedArray = (Object) ByteToDoubleObj(_barray);
break;
default:
HDF5JavaException ex = new HDF5JavaException(
"HDFArray: unsupported Object type: "
+ ArrayDescriptor.NT);
throw (ex);
} // end of switch statement for arrays of boxed objects
default:
HDF5JavaException ex = new HDF5JavaException(
"HDFArray: unknown or unsupported type: "
+ ArrayDescriptor.NT);
throw (ex);
} // end of switch statement for arrays of primitives

while (n < ArrayDescriptor.totalSize) {
oo = ArrayDescriptor.objs[0];
index = n / ArrayDescriptor.bytetoindex[0];
Expand All @@ -524,172 +580,58 @@ else if (ArrayDescriptor.className.equals("java.lang.Long")) {

/* array-ify */
try {
if (ArrayDescriptor.NT == 'J') {
long[] arow = HDFNativeData.byteToLong(n,
ArrayDescriptor.dimlen[ArrayDescriptor.dims],
_barray);
java.lang.reflect.Array
.set(
ArrayDescriptor.objs[ArrayDescriptor.dims - 2],
(ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]),
arow);
n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1];
ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++;
}
else if (ArrayDescriptor.NT == 'I') {
int[] arow = HDFNativeData.byteToInt(n,
ArrayDescriptor.dimlen[ArrayDescriptor.dims],
_barray);
java.lang.reflect.Array
.set(
ArrayDescriptor.objs[ArrayDescriptor.dims - 2],
(ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]),
arow);

n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1];
ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++;
}
else if (ArrayDescriptor.NT == 'S') {
short[] arow = HDFNativeData.byteToShort(n,
ArrayDescriptor.dimlen[ArrayDescriptor.dims],
_barray);
java.lang.reflect.Array
.set(
ArrayDescriptor.objs[ArrayDescriptor.dims - 2],
(ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]),
arow);

n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1];
ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++;
}
else if (ArrayDescriptor.NT == 'B') {
System.arraycopy(_barray, n,
ArrayDescriptor.objs[ArrayDescriptor.dims - 1], 0,
ArrayDescriptor.dimlen[ArrayDescriptor.dims]);
n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1];
}
else if (ArrayDescriptor.NT == 'F') {
float arow[] = HDFNativeData.byteToFloat(n,
ArrayDescriptor.dimlen[ArrayDescriptor.dims],
_barray);
java.lang.reflect.Array
.set(
ArrayDescriptor.objs[ArrayDescriptor.dims - 2],
(ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]),
arow);

n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1];
ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++;
}
else if (ArrayDescriptor.NT == 'D') {
double[] arow = HDFNativeData.byteToDouble(n,
ArrayDescriptor.dimlen[ArrayDescriptor.dims],
_barray);
java.lang.reflect.Array
.set(
ArrayDescriptor.objs[ArrayDescriptor.dims - 2],
(ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]),
arow);

n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1];
ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++;
}
else if (ArrayDescriptor.NT == 'L') {
if (ArrayDescriptor.className.equals("java.lang.Byte")) {
Byte I[] = ByteToByteObj(n,
ArrayDescriptor.dimlen[ArrayDescriptor.dims],
_barray);
java.lang.reflect.Array
.set(
ArrayDescriptor.objs[ArrayDescriptor.dims - 2],
(ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]),
I);

n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1];
ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++;
}
else if (ArrayDescriptor.className
.equals("java.lang.Integer")) {
Integer I[] = ByteToInteger(n,
ArrayDescriptor.dimlen[ArrayDescriptor.dims],
_barray);
java.lang.reflect.Array
.set(
ArrayDescriptor.objs[ArrayDescriptor.dims - 2],
(ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]),
I);

n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1];
ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++;
}
else if (ArrayDescriptor.className
.equals("java.lang.Short")) {
Short I[] = ByteToShort(n,
ArrayDescriptor.dimlen[ArrayDescriptor.dims],
_barray);
java.lang.reflect.Array
.set(
ArrayDescriptor.objs[ArrayDescriptor.dims - 2],
(ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]),
I);

n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1];
ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++;
}
else if (ArrayDescriptor.className
.equals("java.lang.Float")) {
Float I[] = ByteToFloatObj(n,
ArrayDescriptor.dimlen[ArrayDescriptor.dims],
_barray);
java.lang.reflect.Array
.set(
ArrayDescriptor.objs[ArrayDescriptor.dims - 2],
(ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]),
I);

n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1];
ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++;
}
else if (ArrayDescriptor.className
.equals("java.lang.Double")) {
Double I[] = ByteToDoubleObj(n,
ArrayDescriptor.dimlen[ArrayDescriptor.dims],
_barray);
java.lang.reflect.Array
.set(
ArrayDescriptor.objs[ArrayDescriptor.dims - 2],
(ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]),
I);

n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1];
ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++;
}
else if (ArrayDescriptor.className.equals("java.lang.Long")) {
Long I[] = ByteToLongObj(n,
ArrayDescriptor.dimlen[ArrayDescriptor.dims],
_barray);
java.lang.reflect.Array
.set(
ArrayDescriptor.objs[ArrayDescriptor.dims - 2],
(ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]),
I);

n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1];
ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++;
}
else {
HDF5JavaException ex = new HDF5JavaException(
"HDFArray: unsupported Object type: "
+ ArrayDescriptor.NT);
throw (ex);
}
}
else {
HDF5JavaException ex = new HDF5JavaException(
"HDFArray: unknown or unsupported type: "
+ ArrayDescriptor.NT);
throw (ex);
}

Object arow = null;
int mm = m + ArrayDescriptor.dimlen[ArrayDescriptor.dims];
switch (ArrayDescriptor.NT) {
case 'B':
arow = (Object) Arrays.copyOfRange((byte[]) flattenedArray, m, mm);
break;
case 'S':
arow = (Object) Arrays.copyOfRange((short[]) flattenedArray, m, mm);
break;
case 'I':
arow = (Object) Arrays.copyOfRange((int[]) flattenedArray, m, mm);
break;
case 'J':
arow = (Object) Arrays.copyOfRange((long[]) flattenedArray, m, mm);
break;
case 'F':
arow = (Object) Arrays.copyOfRange((float[]) flattenedArray, m, mm);
break;
case 'D':
arow = (Object) Arrays.copyOfRange((double[]) flattenedArray, m, mm);
break;
case 'L':
switch (ArrayDescriptor.className) {
case "java.lang.Byte":
arow = (Object) Arrays.copyOfRange((Byte[])flattenedArray, m, mm);
break;
case "java.lang.Short":
arow = (Object) Arrays.copyOfRange((Short[])flattenedArray, m, mm);
break;
case "java.lang.Integer":
arow = (Object) Arrays.copyOfRange((Integer[])flattenedArray, m, mm);
break;
case "java.lang.Long":
arow = (Object) Arrays.copyOfRange((Long[])flattenedArray, m, mm);
break;
case "java.lang.Float":
arow = (Object) Arrays.copyOfRange((Float[])flattenedArray, m, mm);
break;
case "java.lang.Double":
arow = (Object) Arrays.copyOfRange((Double[])flattenedArray, m, mm);
break;
} // end of switch statement for arrays of boxed numerics
} // end of switch statement for arrays of primitives

java.lang.reflect.Array.set(
ArrayDescriptor.objs[ArrayDescriptor.dims - 2],
(ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]),
arow);
n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1];
ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++;
m = mm;
}
catch (OutOfMemoryError err) {
HDF5JavaException ex = new HDF5JavaException(
Expand Down Expand Up @@ -717,25 +659,15 @@ else if (ArrayDescriptor.className.equals("java.lang.Long")) {
+ (ArrayDescriptor.dimlen[i] - 1) + "?"));
}
}
if (ArrayDescriptor.NT != 'B') {
if (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1] != ArrayDescriptor.dimlen[ArrayDescriptor.dims - 1]) {
throw new java.lang.InternalError(new String(
"HDFArray::arrayify Panic didn't complete all data: currentindex["
+ i + "] = " + ArrayDescriptor.currentindex[i]
+ " (should be " + (ArrayDescriptor.dimlen[i])
+ "?"));
}
}
else {
if (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1] != (ArrayDescriptor.dimlen[ArrayDescriptor.dims - 1] - 1)) {
throw new java.lang.InternalError(new String(
"HDFArray::arrayify Panic didn't complete all data: currentindex["
+ i + "] = " + ArrayDescriptor.currentindex[i]
+ " (should be "
+ (ArrayDescriptor.dimlen[i] - 1) + "?"));
}
if (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1] != ArrayDescriptor.dimlen[ArrayDescriptor.dims - 1]) {
throw new java.lang.InternalError(new String(
"HDFArray::arrayify Panic didn't complete all data: currentindex["
+ i + "] = " + ArrayDescriptor.currentindex[i]
+ " (should be " + (ArrayDescriptor.dimlen[i])
+ "?"));
}


return _theArray;
}

Expand Down Expand Up @@ -944,6 +876,7 @@ class ArrayDescriptor {
static int[] currentindex = null;
static int[] bytetoindex = null;
static int totalSize = 0;
static int totalElements = 0;
static Object[] objs = null;
static char NT = ' '; /* must be B,S,I,L,F,D, else error */
static int NTsize = 0;
Expand Down Expand Up @@ -1052,14 +985,17 @@ else if (css.startsWith("Ljava.lang.String")) {
dimlen[0] = 1;
dimstart[0] = 0;
currentindex[0] = 0;
int elements = 1;
int i;
for (i = 1; i <= dims; i++) {
dimlen[i] = java.lang.reflect.Array.getLength((Object) o);
o = java.lang.reflect.Array.get((Object) o, 0);
objs[i] = o;
dimstart[i] = 0;
currentindex[i] = 0;
elements *= dimlen[i];
}
totalElements = elements;

int j;
int dd;
Expand All @@ -1083,7 +1019,7 @@ public void dumpInfo() {
System.out.println("Class: " + theClass);
System.out.println("NT: " + NT + " NTsize: " + NTsize);
System.out.println("Array has " + dims + " dimensions (" + totalSize
+ " bytes)");
+ " bytes, " + totalElements + " elements)");
int i;
for (i = 0; i <= dims; i++) {
Class tc = objs[i].getClass();
Expand Down
14 changes: 14 additions & 0 deletions release_docs/RELEASE.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,20 @@ Bug Fixes since HDF5-1.12.0 release
(QAK - 2020/05/07)


Java Library:
----------------
- The H5FArray.java class, in which virtually the entire execution time
is spent using the HDFNativeData method that converts from an array
of bytes to an array of the destination Java type.

1. Convert the entire byte array into a 1-d array of the desired type,
rather than performing 1 conversion per row;
2. Use the Java Arrays method copyOfRange to grab the section of the
array from (1) that is desired to be inserted into the destination array.

(PGT,ADB - 2020/12/29, HDFFV-10865)


Tools:
------

Expand Down