Skip to content

Commit 57cb291

Browse files
author
haileyajohnson
authored
Merge pull request #1369 from matakleo/ClassifierServiceProvider
Service Provider for applying enhancements
2 parents 92d0d7d + 7892b2d commit 57cb291

File tree

10 files changed

+131
-34
lines changed

10 files changed

+131
-34
lines changed

‎cdm/core/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ apply from: "$rootDir/gradle/any/protobuf.gradle"
88

99
dependencies {
1010
api enforcedPlatform(project(':netcdf-java-platform'))
11-
implementation 'commons-math:commons-math'
11+
implementation 'org.apache.commons:commons-math3'
1212

1313
testImplementation enforcedPlatform(project(':netcdf-java-testing-platform'))
1414

‎cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java

+16-23
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@
3434
*/
3535
public class VariableDS extends Variable implements VariableEnhanced, EnhanceScaleMissingUnsigned {
3636

37+
38+
3739
/**
3840
* Constructor when there's no underlying variable.
3941
* You must also set the values by doing one of:
@@ -269,8 +271,10 @@ Array convert(Array data, Set<NetcdfDataset.Enhance> enhancements) {
269271
// datatype of the result depends on what enhancements were applied
270272
DataType convertedType = data.getDataType();
271273

274+
272275
// TODO: change to a provider for extensible Enhancements
273276
List<Enhancement> toApply = new ArrayList<>();
277+
274278
if (enhancements.contains(Enhance.ConvertUnsigned) && unsignedConversion != null) {
275279
toApply.add(unsignedConversion);
276280
convertedType = unsignedConversion.getOutType();
@@ -283,15 +287,9 @@ Array convert(Array data, Set<NetcdfDataset.Enhance> enhancements) {
283287
toApply.add(scaleOffset);
284288
convertedType = scaleOffset.getScaledOffsetType();
285289
}
286-
if (enhancements.contains(Enhance.ApplyStandardizer) && standardizer != null) {
287-
toApply.add(standardizer);
288-
}
289-
if (enhancements.contains(Enhance.ApplyNormalizer) && normalizer != null) {
290-
toApply.add(normalizer);
291-
}
292-
if (enhancements.contains(Enhance.ApplyClassifier) && classifier != null) {
293-
toApply.add(classifier);
294-
}
290+
291+
toApply.addAll(loadedEnhancements);
292+
295293

296294
double[] dataArray = (double[]) data.get1DJavaArray(DataType.DOUBLE);
297295

@@ -866,9 +864,8 @@ public Array convert(Array in, boolean convertUnsigned, boolean applyScaleOffset
866864
// TODO make immutable in version 6
867865
private UnsignedConversion unsignedConversion;
868866
private ScaleOffset scaleOffset;
869-
private Standardizer standardizer;
870-
private Normalizer normalizer;
871-
private Classifier classifier;
867+
private List<Enhancement> loadedEnhancements = new ArrayList<>();
868+
872869
private ConvertMissing convertMissing;
873870
private Set<Enhance> enhanceMode = EnumSet.noneOf(Enhance.class); // The set of enhancements that were made.
874871

@@ -935,18 +932,14 @@ private void createEnhancements() {
935932
}
936933
this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType;
937934
}
938-
Attribute standardizerAtt = findAttribute(CDM.STANDARDIZE);
939-
if (standardizerAtt != null && this.enhanceMode.contains(Enhance.ApplyStandardizer) && dataType.isFloatingPoint()) {
940-
this.standardizer = Standardizer.createFromVariable(this);
941-
}
942-
Attribute normalizerAtt = findAttribute(CDM.NORMALIZE);
943-
if (normalizerAtt != null && this.enhanceMode.contains(Enhance.ApplyNormalizer) && dataType.isFloatingPoint()) {
944-
this.normalizer = Normalizer.createFromVariable(this);
945-
}
946-
Attribute classifierAtt = findAttribute(CDM.CLASSIFY);
947-
if (classifierAtt != null && this.enhanceMode.contains(Enhance.ApplyClassifier) && dataType.isNumeric()) {
948-
this.classifier = Classifier.createFromVariable(this);
935+
for (Enhance enhance : this.enhanceMode) {
936+
for (EnhancementProvider service : ServiceLoader.load(EnhancementProvider.class)) {
937+
if (service.appliesTo(enhance, this.attributes(), dataType)) {
938+
loadedEnhancements.add(service.create(this));
939+
}
940+
}
949941
}
942+
950943
}
951944

952945
public Builder<?> toBuilder() {

‎cdm/core/src/main/java/ucar/nc2/filter/Classifier.java

+26-3
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,28 @@
11
package ucar.nc2.filter;
22

3-
43
import ucar.ma2.Array;
54
import ucar.ma2.IndexIterator;
6-
import ucar.nc2.Variable;
75
import ucar.nc2.constants.CDM;
86
import ucar.nc2.Attribute;
7+
import ucar.nc2.dataset.NetcdfDataset.Enhance;
8+
import ucar.nc2.dataset.VariableDS;
99
import ucar.nc2.util.Misc;
1010
import java.util.ArrayList;
1111
import java.util.List;
12+
import ucar.ma2.*;
13+
import ucar.nc2.*;
1214

1315

1416
public class Classifier implements Enhancement {
17+
18+
1519
private String[] AttCat;
1620
private List<int[]> rules = new ArrayList<>();
1721

22+
private static String name = "Classifier";
23+
24+
25+
1826
public Classifier() {
1927
this.AttCat = new String[0];
2028
this.rules = new ArrayList<>();
@@ -27,7 +35,8 @@ public Classifier(String[] AttCat) {
2735
}
2836

2937
// Factory method to create a Classifier from a Variable
30-
public static Classifier createFromVariable(Variable var) {
38+
39+
public static Classifier createFromVariable(VariableDS var) {
3140
List<Attribute> attributes = var.attributes().getAttributes();
3241

3342
for (Attribute attribute : attributes) {
@@ -116,4 +125,18 @@ public static int[] stringToIntArray(String str) {
116125
return intArray;
117126
}
118127

128+
public static class Provider implements EnhancementProvider {
129+
@Override
130+
public boolean appliesTo(Enhance enhance, AttributeContainer attributes, DataType dt) {
131+
return enhance == Enhance.ApplyClassifier && attributes.findAttribute(CDM.CLASSIFY) != null && dt.isNumeric();
132+
}
133+
134+
@Override
135+
public Classifier create(VariableDS var) {
136+
return createFromVariable(var);
137+
}
138+
}
139+
119140
}
141+
142+

‎cdm/core/src/main/java/ucar/nc2/filter/EnhancementProvider.java

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright (c) 2021 University Corporation for Atmospheric Research/Unidata
3+
* See LICENSE for license information.
4+
*/
5+
6+
package ucar.nc2.filter;
7+
8+
import ucar.ma2.*;
9+
import ucar.nc2.*;
10+
import ucar.nc2.dataset.NetcdfDataset.Enhance;
11+
import ucar.nc2.dataset.VariableDS;
12+
13+
14+
/**
15+
* A Service Provider of {@link Enhancement}.
16+
*/
17+
public interface EnhancementProvider {
18+
19+
boolean appliesTo(Enhance enhance, AttributeContainer attributes, DataType dt);
20+
21+
Enhancement create(VariableDS var);
22+
23+
24+
}
25+
26+

‎cdm/core/src/main/java/ucar/nc2/filter/Normalizer.java

+25-2
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,23 @@
33
import java.io.IOException;
44
import java.util.HashMap;
55
import java.util.Map;
6-
7-
import org.apache.commons.math.stat.descriptive.SummaryStatistics;
6+
import ucar.nc2.constants.CDM;
7+
import java.util.Set;
8+
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
89
import ucar.ma2.Array;
910
import ucar.ma2.DataType;
1011
import ucar.ma2.IndexIterator;
12+
import ucar.nc2.AttributeContainer;
13+
import ucar.nc2.Variable;
14+
import ucar.nc2.dataset.NetcdfDataset.Enhance;
1115
import ucar.nc2.dataset.VariableDS;
1216

1317
public class Normalizer implements Enhancement {
1418

1519
private final ScaleOffset scaleOffset;
1620
private final double minimum;
1721
private final double range; // maximum - minimum
22+
private static String name = "Normalizer";
1823

1924
public static Normalizer createFromVariable(VariableDS var) {
2025
try {
@@ -72,4 +77,22 @@ public double getRange() {
7277
return range;
7378
}
7479

80+
public static class Provider implements EnhancementProvider {
81+
82+
83+
@Override
84+
public boolean appliesTo(Enhance enhance, AttributeContainer attributes, DataType dt) {
85+
return enhance == Enhance.ApplyNormalizer && attributes.findAttribute(CDM.NORMALIZE) != null
86+
&& dt.isFloatingPoint();
87+
}
88+
89+
@Override
90+
public Normalizer create(VariableDS var) {
91+
return Normalizer.createFromVariable(var);
92+
}
93+
94+
95+
96+
}
97+
7598
}

‎cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java

+3
Original file line numberDiff line numberDiff line change
@@ -290,5 +290,8 @@ public int getId() {
290290
public Filter create(Map<String, Object> properties) {
291291
return new ScaleOffset(properties);
292292
}
293+
293294
}
295+
296+
294297
}

‎cdm/core/src/main/java/ucar/nc2/filter/Standardizer.java

+23-1
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,23 @@
33
import java.io.IOException;
44
import java.util.HashMap;
55
import java.util.Map;
6+
import ucar.nc2.constants.CDM;
67

7-
import org.apache.commons.math.stat.descriptive.SummaryStatistics;
8+
import java.util.Set;
9+
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
810
import ucar.ma2.Array;
911
import ucar.ma2.DataType;
1012
import ucar.ma2.IndexIterator;
13+
import ucar.nc2.AttributeContainer;
14+
import ucar.nc2.dataset.NetcdfDataset.Enhance;
1115
import ucar.nc2.dataset.VariableDS;
1216

1317
public class Standardizer implements Enhancement {
1418

1519
private final ScaleOffset scaleOffset;
1620
private final double mean;
1721
private final double stdDev;
22+
private static String name = "Standardizer";
1823

1924
public static Standardizer createFromVariable(VariableDS var) {
2025
try {
@@ -71,5 +76,22 @@ public double getMean() {
7176
public double getStdDev() {
7277
return stdDev;
7378
}
79+
80+
81+
82+
public static class Provider implements EnhancementProvider {
83+
84+
85+
@Override
86+
public boolean appliesTo(Enhance enhance, AttributeContainer attributes, DataType dt) {
87+
return enhance == Enhance.ApplyStandardizer && attributes.findAttribute(CDM.STANDARDIZE) != null
88+
&& dt.isFloatingPoint();
89+
}
90+
91+
@Override
92+
public Standardizer create(VariableDS var) {
93+
return createFromVariable(var);
94+
}
95+
}
7496
}
7597

‎cdm/core/src/main/resources/META-INF/services/ucar.nc2.filter.EnhancementProvider

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
ucar.nc2.filter.Classifier$Provider
2+
ucar.nc2.filter.Standardizer$Provider
3+
ucar.nc2.filter.Normalizer$Provider
4+
5+
6+

‎cdm/core/src/test/java/ucar/nc2/ncml/TestEnhanceClassifier.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import ucar.nc2.NetcdfFile;
1313
import ucar.nc2.Variable;
1414
import ucar.nc2.dataset.NetcdfDatasets;
15+
import ucar.nc2.dataset.VariableDS;
1516
import ucar.nc2.filter.Classifier;
1617
import ucar.unidata.util.test.TestDir;
1718

@@ -33,7 +34,7 @@ public void testEnhanceClassifier_Ints() throws IOException {
3334
assertThat((Object) classifySpecs).isNotNull();
3435
assertThat(!classifySpecs.attributes().isEmpty()).isTrue();
3536
Array Data = classifySpecs.read();
36-
Classifier classifier = Classifier.createFromVariable(classifySpecs);
37+
Classifier classifier = Classifier.createFromVariable((VariableDS) classifySpecs);
3738
int[] ClassifiedArray = classifier.classifyWithAttributes(Data);
3839
assertThat(nearlyEquals(Array.makeFromJavaArray(ClassifiedArray), DATA_mixNumbers)).isTrue();
3940

@@ -53,7 +54,7 @@ public void testEnhanceClassifier_Floats() throws IOException {
5354
assertThat((Object) classifySpecs).isNotNull();
5455
assertThat(!classifySpecs.attributes().isEmpty()).isTrue();
5556
Array Data = classifySpecs.read();
56-
Classifier classifier = Classifier.createFromVariable(classifySpecs);
57+
Classifier classifier = Classifier.createFromVariable((VariableDS) classifySpecs);
5758
int[] ClassifiedArray = classifier.classifyWithAttributes(Data);
5859
assertThat(nearlyEquals(Array.makeFromJavaArray(ClassifiedArray), DATA_mixNumbers)).isTrue();
5960

@@ -73,7 +74,7 @@ public void testEnhanceClassifier_classification() throws IOException {
7374
assertThat((Object) classifySpecs).isNotNull();
7475
assertThat(!classifySpecs.attributes().isEmpty()).isTrue();
7576
Array Data = classifySpecs.read();
76-
Classifier classifier = Classifier.createFromVariable(classifySpecs);
77+
Classifier classifier = Classifier.createFromVariable((VariableDS) classifySpecs);
7778
int[] ClassifiedArray = classifier.classifyWithAttributes(Data);
7879
assertThat(nearlyEquals(Array.makeFromJavaArray(ClassifiedArray), CLASSIFICATION_TEST)).isTrue();
7980

‎netcdf-java-platform/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ dependencies {
3737
api 'com.beust:jcommander:1.78'
3838

3939
// cdm-core
40-
api 'commons-math:commons-math:1.2'
40+
api 'org.apache.commons:commons-math3:3.6.1'
4141

4242
// cdm-grib
4343
api 'edu.ucar:jj2000:5.4'

0 commit comments

Comments
 (0)