Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[5.x] work on filters package #966

Merged
merged 5 commits into from
Aug 22, 2022
Merged

[5.x] work on filters package #966

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
295cd18
changes to the filter api; port hdf5; docs updates
Dec 10, 2021
2492582
fix filters for hdf5
Jan 5, 2022
ba53441
spotless and clean up
Aug 19, 2022
e8edf43
Update cdm/core/src/main/java/ucar/nc2/filter/Filters.java
Aug 22, 2022
3a60b95
Update cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5tiledLayo…
Aug 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions cdm/core/src/main/java/ucar/nc2/filter/Blosc.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,22 @@
// TODO: Still to be implemented
public class Blosc extends Filter {

private static final String name = "blosc";

private static final int id = 32001;

public Blosc(Map<String, Object> properties) {}

@Override
public String getName() {
return name;
}

@Override
public int getId() {
return id;
}

@Override
public byte[] encode(byte[] dataIn) {
return new byte[0];
Expand All @@ -25,10 +39,6 @@ public byte[] decode(byte[] dataIn) {

public static class Provider implements FilterProvider {

private static final String name = "blosc";

private static final int id = -1; // not yet implemented by id

@Override
public String getName() {
return name;
Expand Down
45 changes: 26 additions & 19 deletions cdm/core/src/main/java/ucar/nc2/filter/Checksum32.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,16 @@
public class Checksum32 extends Filter {

public enum CType {
FLETCHER, ADLER, CRC
FLETCHER("fletcher32", 3), ADLER("adler32", -1), CRC("crc32", -1);
tdrwenski marked this conversation as resolved.
Show resolved Hide resolved

private final String name;

private final int id;

CType(String name, int id) {
this.name = name;
this.id = id;
}
}

private static final int nbytes = 4; // number of bytes in the checksum
Expand All @@ -39,6 +48,16 @@ public Checksum32(CType type) {
this(type, ByteOrder.LITTLE_ENDIAN);
}

@Override
public String getName() {
return type.name;
}

@Override
public int getId() {
return type.id;
}

@Override
public byte[] encode(byte[] dataIn) {
// create a checksum
Expand Down Expand Up @@ -112,18 +131,14 @@ public void update(byte[] b, int off, int len) {

public static class Fletcher32Provider implements FilterProvider {

private static final String name = "fletcher32";

private static final int id = -1;

@Override
public String getName() {
return name;
return CType.FLETCHER.name;
}

@Override
public int getId() {
return id;
return CType.FLETCHER.id;
}

@Override
Expand All @@ -134,18 +149,14 @@ public Filter create(Map<String, Object> properties) {

public static class Adler32Provider implements FilterProvider {

private static final String name = "adler32";

private static final int id = -1; // not yet implemented by id

@Override
public String getName() {
return name;
return CType.ADLER.name;
}

@Override
public int getId() {
return id;
return CType.ADLER.id;
}

@Override
Expand All @@ -156,18 +167,14 @@ public Filter create(Map<String, Object> properties) {

public static class CRC32Provider implements FilterProvider {

private static final String name = "crc32";

private static final int id = -1; // not yet implemented by id

@Override
public String getName() {
return name;
return CType.CRC.name;
}

@Override
public int getId() {
return id;
return CType.CRC.id;
}

@Override
Expand Down
18 changes: 14 additions & 4 deletions cdm/core/src/main/java/ucar/nc2/filter/Deflate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@
*/
public class Deflate extends Filter {

private static final String name = "zlib";

private static final int id = 1;

private final int clevel; // compression level

public Deflate(Map<String, Object> properties) {
Expand All @@ -39,6 +43,16 @@ private void validateLevel() {
}
}

@Override
public String getName() {
return name;
}

@Override
public int getId() {
return id;
}

@Override
public byte[] encode(byte[] dataIn) throws IOException {
ByteArrayOutputStream os = new ByteArrayOutputStream(dataIn.length);
Expand Down Expand Up @@ -70,10 +84,6 @@ public byte[] decode(byte[] dataIn) throws IOException {

public static class Provider implements FilterProvider {

private static final String name = "zlib";

private static final int id = -1; // not yet implemented by id

@Override
public String getName() {
return name;
Expand Down
11 changes: 11 additions & 0 deletions cdm/core/src/main/java/ucar/nc2/filter/Filter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,22 @@
package ucar.nc2.filter;

import java.io.IOException;
import java.nio.ByteOrder;
import java.util.Formatter;
import java.util.Map;

public abstract class Filter {

public abstract String getName();

public abstract int getId();

public abstract byte[] encode(byte[] dataIn) throws IOException;

public abstract byte[] decode(byte[] dataIn) throws IOException;

public String toString() {
Formatter f = new Formatter();
return f.format("Name: %s, ID: %d", getName(), getId()).toString();
}
}
1 change: 0 additions & 1 deletion cdm/core/src/main/java/ucar/nc2/filter/FilterProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ default boolean canProvide(int id) {
return id == getId();
}


/**
* Create a {@link ucar.nc2.filter.Filter} of the correct type
*
Expand Down
73 changes: 65 additions & 8 deletions cdm/core/src/main/java/ucar/nc2/filter/Filters.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,38 +10,95 @@

public class Filters {

/**
* Set of common properties used by Filters
*/
public class Keys {
/**
* key for a String id/name (e.g. Zarr filters)
*/
public static final String NAME = "id";
/**
* key for a numeric id (e.g. HDF5 filters)
*/
public static final String ID = "nid";
/**
* key for property indicating whether filter is optional,
* i.e. whether it should continue or throw if it fails
*/
public static final String OPTIONAL = "optional";
/**
* key mapping to client data used by filter
*/
public static final String DATA = "data";
/**
* key mapping to element size
*/
public static final String ELEM_SIZE = "elementsize";
}

private static NullFilter nullFilter = new NullFilter();

/**
* Create a filter object for the given property list
* Create a filter object matching the provided id
*
* @param properties
* @return A filter object with the provided properties
* @return A filter object with the given properties
* @throws UnknownFilterException
*/
public static Filter getFilterByName(Map<String, Object> properties) throws UnknownFilterException {
public static Filter getFilter(Map<String, Object> properties) throws UnknownFilterException {
tdrwenski marked this conversation as resolved.
Show resolved Hide resolved
if (properties == null) {
return nullFilter;
}
String name = (String) properties.get("id");
if (name == null || name.isEmpty()) {
// read id properties
String name = (String) properties.get(Keys.NAME);
Object oid = properties.get(Keys.ID);
// if no id or name, return null filter
if ((name == null || name.isEmpty()) && !(oid instanceof Number)) {
return nullFilter;
}

// look for dynamically loaded filters
// try by name first
if (name != null && !name.isEmpty()) {
// look for dynamically loaded filters by name
for (FilterProvider fp : ServiceLoader.load(FilterProvider.class)) {
if (fp.canProvide(name)) {
return fp.create(properties);
}
}
}

// try by id next
int id = ((Short) oid).intValue();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe we should check that oid instanceof Short before casting here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we already checked that it's a Number, I think any Number type can cast to a Short without an exception

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah, maybe I am overthinking it here, but could it happen that a filter does have a name but no provider for that name? then you reach this code without having checked if the oid instanceof Number

// look for dynamically loaded filters by id
for (FilterProvider fp : ServiceLoader.load(FilterProvider.class)) {
if (fp.canProvide(name)) {
if (fp.canProvide(id)) {
return fp.create(properties);
}
}
throw new UnknownFilterException(name);
// final fallback
throw new UnknownFilterException(id);
}

/**
* A filter which passes data through unchanged
*/
private static class NullFilter extends Filter {

private static final String name = "null";

private static final int id = 0;

@Override
public String getName() {
return name;
}

@Override
public int getId() {
return id;
}

@Override
public byte[] encode(byte[] dataIn) {
return dataIn;
Expand Down
17 changes: 13 additions & 4 deletions cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@
*/
public class ScaleOffset extends Filter {

private static final String name = "fixedscaleoffset";

private static final int id = 6;

// maps numeric zarr datatypes to CDM datatypes
private static Map<String, DataType> dTypeMap;

Expand Down Expand Up @@ -69,6 +73,15 @@ public ScaleOffset(Map<String, Object> properties) {
astypeOrder = parseByteOrder(aType, dtypeOrder);
}

@Override
public String getName() {
return name;
}

@Override
public int getId() {
return id;
}

@Override
public byte[] encode(byte[] dataIn) {
Expand Down Expand Up @@ -271,10 +284,6 @@ private byte[] arrayToBytes(Array arr, DataType type, ByteOrder order) {

public static class Provider implements FilterProvider {

private static final String name = "fixedscaleoffset";

private static final int id = -1; // not yet implemented by id

@Override
public String getName() {
return name;
Expand Down
20 changes: 15 additions & 5 deletions cdm/core/src/main/java/ucar/nc2/filter/Shuffle.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,31 @@
*/
public class Shuffle extends Filter {

private static final String name = "shuffle";

private static final int id = 2;

private int elemSize;
private static final int DEFAULT_SIZE = 4;

public Shuffle(Map<String, Object> properties) {
try {
elemSize = (int) properties.get("elementsize");
elemSize = (int) properties.get(Filters.Keys.ELEM_SIZE);
} catch (Exception ex) {
elemSize = DEFAULT_SIZE;
}
}

@Override
public String getName() {
return name;
}

@Override
public int getId() {
return id;
}

@Override
public byte[] encode(byte[] dataIn) {
if (dataIn.length % elemSize != 0 || elemSize <= 1) {
Expand Down Expand Up @@ -69,10 +83,6 @@ public byte[] decode(byte[] dataIn) {

public static class Provider implements FilterProvider {

private static final String name = "shuffle";

private static final int id = -1; // not yet implemented by id

@Override
public String getName() {
return name;
Expand Down
Loading