Skip to content

Commit fe00c9b

Browse files
rzo1rzo1
committed
OPENNLP-1633 - Remove dependency towards jackson-databind in opennlp-dl module
1 parent 74c7d52 commit fe00c9b

File tree

6 files changed

+226
-56
lines changed

6 files changed

+226
-56
lines changed

‎NOTICE

-6
Original file line numberDiff line numberDiff line change
@@ -93,9 +93,3 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
9393
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
9494
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
9595
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
96-
97-
============================================================================
98-
99-
jackson-databind
100-
https://github.com/FasterXML/jackson-databind
101-
The Apache Software License, Version 2.0

‎opennlp-brat-annotator/pom.xml

-7
Original file line numberDiff line numberDiff line change
@@ -61,13 +61,6 @@
6161
<artifactId>jackson-databind</artifactId>
6262
<version>${jackson.version}</version>
6363
<scope>runtime</scope>
64-
<exclusions>
65-
<!-- Byte-Buddy became a dependency by accident - TODO remove it with update version > 2.17.0 -->
66-
<exclusion>
67-
<groupId>net.bytebuddy</groupId>
68-
<artifactId>byte-buddy</artifactId>
69-
</exclusion>
70-
</exclusions>
7164
</dependency>
7265

7366
<dependency>

‎opennlp-dl/pom.xml

-12
Original file line numberDiff line numberDiff line change
@@ -41,18 +41,6 @@
4141
<artifactId>onnxruntime</artifactId>
4242
<version>${onnxruntime.version}</version>
4343
</dependency>
44-
<dependency>
45-
<groupId>com.fasterxml.jackson.core</groupId>
46-
<artifactId>jackson-databind</artifactId>
47-
<version>${jackson.version}</version>
48-
<exclusions>
49-
<!-- Byte-Buddy became a dependency by accident - TODO remove it with update version > 2.17.0 -->
50-
<exclusion>
51-
<groupId>net.bytebuddy</groupId>
52-
<artifactId>byte-buddy</artifactId>
53-
</exclusion>
54-
</exclusions>
55-
</dependency>
5644
<dependency>
5745
<groupId>org.slf4j</groupId>
5846
<artifactId>slf4j-api</artifactId>

‎opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerConfig.java

+27-6
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,39 @@
1818
package opennlp.dl.doccat;
1919

2020
import java.util.Collections;
21+
import java.util.HashMap;
2122
import java.util.Map;
23+
import java.util.Objects;
24+
import java.util.regex.Matcher;
25+
import java.util.regex.Pattern;
2226

23-
public class DocumentCategorizerConfig {
27+
public record DocumentCategorizerConfig(Map<String, String> id2label) {
2428

25-
private Map<String, String> id2label;
29+
private static final Pattern ID_TO_LABEL_PATTERN = Pattern.compile("\"id2label\"\\s*:\\s*\\{(.*?)\\}", Pattern.DOTALL);
30+
private static final Pattern ENTRY_PATTERN = Pattern.compile("\"([^\"]+)\"\\s*:\\s*\"(.*?)\"");
2631

27-
public Map<String, String> getId2label() {
32+
@Override
33+
public Map<String, String> id2label() {
2834
return Collections.unmodifiableMap(id2label);
2935
}
3036

31-
public void setId2label(Map<String, String> id2label) {
32-
this.id2label = id2label;
33-
}
37+
public static DocumentCategorizerConfig fromJson(String json) {
38+
Objects.requireNonNull(json, "json must not be null");
39+
40+
final Map<String, String> id2label = new HashMap<>();
41+
final Matcher matcher = ID_TO_LABEL_PATTERN.matcher(json);
42+
43+
if (matcher.find()) {
44+
final String id2labelContent = matcher.group(1);
45+
final Matcher entryMatcher = ENTRY_PATTERN.matcher(id2labelContent);
3446

47+
while (entryMatcher.find()) {
48+
final String key = entryMatcher.group(1);
49+
final String value = entryMatcher.group(2);
50+
id2label.put(key, value);
51+
}
52+
}
53+
54+
return new DocumentCategorizerConfig(id2label);
55+
}
3556
}

‎opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java

+17-25
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,13 @@
3030
import java.util.Set;
3131
import java.util.SortedMap;
3232
import java.util.TreeMap;
33+
3334
import java.util.stream.IntStream;
3435

3536
import ai.onnxruntime.OnnxTensor;
3637
import ai.onnxruntime.OrtEnvironment;
3738
import ai.onnxruntime.OrtException;
3839
import ai.onnxruntime.OrtSession;
39-
import com.fasterxml.jackson.databind.DeserializationFeature;
40-
import com.fasterxml.jackson.databind.ObjectMapper;
4140
import org.slf4j.Logger;
4241
import org.slf4j.LoggerFactory;
4342

@@ -67,16 +66,15 @@ public class DocumentCategorizerDL extends AbstractDL implements DocumentCategor
6766
/**
6867
* Instantiates a {@link DocumentCategorizer document categorizer} using ONNX models.
6968
*
70-
* @param model The ONNX model file.
71-
* @param vocabulary The model file's vocabulary file.
72-
* @param categories The categories.
69+
* @param model The ONNX model file.
70+
* @param vocabulary The model file's vocabulary file.
71+
* @param categories The categories.
7372
* @param classificationScoringStrategy Implementation of {@link ClassificationScoringStrategy} used
7473
* to calculate the classification scores given the score of each
7574
* individual document part.
76-
* @param inferenceOptions {@link InferenceOptions} to control the inference.
77-
*
75+
* @param inferenceOptions {@link InferenceOptions} to control the inference.
7876
* @throws OrtException Thrown if the {@code model} cannot be loaded.
79-
* @throws IOException Thrown if errors occurred loading the {@code model} or {@code vocabulary}.
77+
* @throws IOException Thrown if errors occurred loading the {@code model} or {@code vocabulary}.
8078
*/
8179
public DocumentCategorizerDL(File model, File vocabulary, Map<Integer, String> categories,
8280
ClassificationScoringStrategy classificationScoringStrategy,
@@ -102,21 +100,20 @@ public DocumentCategorizerDL(File model, File vocabulary, Map<Integer, String> c
102100
/**
103101
* Instantiates a {@link DocumentCategorizer document categorizer} using ONNX models.
104102
*
105-
* @param model The ONNX model file.
106-
* @param vocabulary The model file's vocabulary file.
107-
* @param config The model's config file. The file will be used to determine the classification categories.
103+
* @param model The ONNX model file.
104+
* @param vocabulary The model file's vocabulary file.
105+
* @param config The model's config file. The file will be used to determine the classification categories.
108106
* @param classificationScoringStrategy Implementation of {@link ClassificationScoringStrategy} used
109107
* to calculate the classification scores given the score of each
110108
* individual document part.
111-
* @param inferenceOptions {@link InferenceOptions} to control the inference.
112-
*
109+
* @param inferenceOptions {@link InferenceOptions} to control the inference.
113110
* @throws OrtException Thrown if the {@code model} cannot be loaded.
114-
* @throws IOException Thrown if errors occurred loading the {@code model} or {@code vocabulary}.
111+
* @throws IOException Thrown if errors occurred loading the {@code model} or {@code vocabulary}.
115112
*/
116113
public DocumentCategorizerDL(File model, File vocabulary, File config,
117114
ClassificationScoringStrategy classificationScoringStrategy,
118115
InferenceOptions inferenceOptions)
119-
throws IOException, OrtException {
116+
throws IOException, OrtException {
120117

121118
this.env = OrtEnvironment.getEnvironment();
122119

@@ -175,7 +172,7 @@ public double[] categorize(String[] strings) {
175172
logger.error("Unload to perform document classification inference", ex);
176173
}
177174

178-
return new double[]{};
175+
return new double[] {};
179176

180177
}
181178

@@ -315,6 +312,7 @@ private List<Tokens> tokenize(final String text) {
315312

316313
/**
317314
* Applies softmax to an array of values.
315+
*
318316
* @param input An array of values.
319317
* @return The output array.
320318
*/
@@ -346,18 +344,12 @@ private int maxIndex(double[] arr) {
346344
}
347345

348346
private Map<Integer, String> readCategoriesFromFile(File config) throws IOException {
349-
350-
final String json = new String(Files.readAllBytes(config.toPath()));
351-
352-
final ObjectMapper objectMapper = new ObjectMapper();
353-
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
354-
355347
final DocumentCategorizerConfig documentCategorizerConfig =
356-
objectMapper.readValue(json, DocumentCategorizerConfig.class);
348+
DocumentCategorizerConfig.fromJson(new String(Files.readAllBytes(config.toPath())));
357349

358350
final Map<Integer, String> categories = new HashMap<>();
359-
for (final String key : documentCategorizerConfig.getId2label().keySet()) {
360-
categories.put(Integer.valueOf(key), documentCategorizerConfig.getId2label().get(key));
351+
for (final String key : documentCategorizerConfig.id2label().keySet()) {
352+
categories.put(Integer.valueOf(key), documentCategorizerConfig.id2label().get(key));
361353
}
362354

363355
return categories;

‎opennlp-dl/src/test/java/opennlp/dl/doccat/DocumentCategorizerConfigTest.java

+182
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package opennlp.dl.doccat;
18+
19+
import org.junit.jupiter.api.Test;
20+
21+
import static org.junit.jupiter.api.Assertions.assertEquals;
22+
import static org.junit.jupiter.api.Assertions.assertNotNull;
23+
24+
25+
public class DocumentCategorizerConfigTest {
26+
27+
@Test
28+
public void testId2LabelsFromJsonPrettyValid() {
29+
final String json = """
30+
{
31+
"_num_labels": 5,
32+
"architectures": [
33+
"BertForSequenceClassification"
34+
],
35+
"attention_probs_dropout_prob": 0.1,
36+
"directionality": "bidi",
37+
"finetuning_task": "sentiment-analysis",
38+
"hidden_act": "gelu",
39+
"hidden_dropout_prob": 0.1,
40+
"hidden_size": 768,
41+
"id2label": {
42+
"0": "1 star",
43+
"1": "2 stars",
44+
"2": "3 stars",
45+
"3": "4 stars",
46+
"4": "5 stars"
47+
},
48+
"initializer_range": 0.02,
49+
"intermediate_size": 3072,
50+
"label2id": {
51+
"1 star": 0,
52+
"2 stars": 1,
53+
"3 stars": 2,
54+
"4 stars": 3,
55+
"5 stars": 4
56+
},
57+
"layer_norm_eps": 1e-12,
58+
"max_position_embeddings": 512,
59+
"model_type": "bert",
60+
"num_attention_heads": 12,
61+
"num_hidden_layers": 12,
62+
"output_past": true,
63+
"pad_token_id": 0,
64+
"pooler_fc_size": 768,
65+
"pooler_num_attention_heads": 12,
66+
"pooler_num_fc_layers": 3,
67+
"pooler_size_per_head": 128,
68+
"pooler_type": "first_token_transform",
69+
"type_vocab_size": 2,
70+
"vocab_size": 105879
71+
}
72+
""";
73+
74+
final DocumentCategorizerConfig config = DocumentCategorizerConfig.fromJson(json);
75+
assertNotNull(config);
76+
assertEquals(5, config.id2label().size());
77+
assertEquals("1 star", config.id2label().get("0"));
78+
assertEquals("2 stars", config.id2label().get("1"));
79+
assertEquals("3 stars", config.id2label().get("2"));
80+
assertEquals("4 stars", config.id2label().get("3"));
81+
assertEquals("5 stars", config.id2label().get("4"));
82+
}
83+
84+
@Test
85+
public void testId2LabelsFromJsonUglyValid() {
86+
final String json = """
87+
{"_num_labels":5,"architectures":["BertForSequenceClassification"],"attention_probs_dropout_prob":0.1,"directionality":"bidi","finetuning_task":"sentiment-analysis",
88+
"hidden_act":"gelu","hidden_dropout_prob":0.1,"hidden_size":768,"id2label":{"0":"1 star","1":"2 stars","2":"3 stars","3":"4 stars","4":"5 stars"},"initializer_range":0.02,
89+
"intermediate_size":3072,"label2id":{"1 star":0,"2 stars":1,"3 stars":2,"4 stars":3,"5 stars":4},"layer_norm_eps":1e-12,"max_position_embeddings":512,"model_type":"bert",
90+
"num_attention_heads":12,"num_hidden_layers":12,"output_past":true,"pad_token_id":0,"pooler_fc_size":768,"pooler_num_attention_heads":12,"pooler_num_fc_layers":3,
91+
"pooler_size_per_head":128,"pooler_type":"first_token_transform","type_vocab_size":2,"vocab_size":105879}
92+
""";
93+
94+
final DocumentCategorizerConfig config = DocumentCategorizerConfig.fromJson(json);
95+
assertNotNull(config);
96+
assertEquals(5, config.id2label().size());
97+
assertEquals("1 star", config.id2label().get("0"));
98+
assertEquals("2 stars", config.id2label().get("1"));
99+
assertEquals("3 stars", config.id2label().get("2"));
100+
assertEquals("4 stars", config.id2label().get("3"));
101+
assertEquals("5 stars", config.id2label().get("4"));
102+
}
103+
104+
@Test
105+
public void testId2LabelsFromJsonNoValues() {
106+
final String json = """
107+
{"_num_labels":5,"architectures":["BertForSequenceClassification"],"attention_probs_dropout_prob":0.1,"directionality":"bidi","finetuning_task":"sentiment-analysis",
108+
"hidden_act":"gelu","hidden_dropout_prob":0.1,"hidden_size":768,"layer_norm_eps":1e-12,"max_position_embeddings":512,"model_type":"bert",
109+
"num_attention_heads":12,"num_hidden_layers":12,"output_past":true,"pad_token_id":0,"pooler_fc_size":768,"pooler_num_attention_heads":12,"pooler_num_fc_layers":3,
110+
"pooler_size_per_head":128,"pooler_type":"first_token_transform","type_vocab_size":2,"vocab_size":105879}
111+
""";
112+
113+
final DocumentCategorizerConfig config = DocumentCategorizerConfig.fromJson(json);
114+
assertNotNull(config);
115+
assertEquals(0, config.id2label().size());
116+
}
117+
118+
@Test
119+
public void testId2LabelsFromJsonEmptyInput() {
120+
final String json = "";
121+
final DocumentCategorizerConfig config = DocumentCategorizerConfig.fromJson(json);
122+
assertNotNull(config);
123+
assertEquals(0, config.id2label().size());
124+
}
125+
126+
@Test
127+
public void testId2LabelsFromJsonPrettyIdIsNotANumberValid() {
128+
final String json = """
129+
{
130+
"_num_labels": 5,
131+
"architectures": [
132+
"BertForSequenceClassification"
133+
],
134+
"attention_probs_dropout_prob": 0.1,
135+
"directionality": "bidi",
136+
"finetuning_task": "sentiment-analysis",
137+
"hidden_act": "gelu",
138+
"hidden_dropout_prob": 0.1,
139+
"hidden_size": 768,
140+
"id2label": {
141+
"a0": "1 star",
142+
"a1": "2 stars",
143+
"a2": "3 stars",
144+
"a3": "4 stars",
145+
"a4": "5 stars"
146+
},
147+
"initializer_range": 0.02,
148+
"intermediate_size": 3072,
149+
"label2id": {
150+
"1 star": "a0",
151+
"2 stars": "a1",
152+
"3 stars": "a2",
153+
"4 stars": "a3",
154+
"5 stars": "a4"
155+
},
156+
"layer_norm_eps": 1e-12,
157+
"max_position_embeddings": 512,
158+
"model_type": "bert",
159+
"num_attention_heads": 12,
160+
"num_hidden_layers": 12,
161+
"output_past": true,
162+
"pad_token_id": 0,
163+
"pooler_fc_size": 768,
164+
"pooler_num_attention_heads": 12,
165+
"pooler_num_fc_layers": 3,
166+
"pooler_size_per_head": 128,
167+
"pooler_type": "first_token_transform",
168+
"type_vocab_size": 2,
169+
"vocab_size": 105879
170+
}
171+
""";
172+
173+
final DocumentCategorizerConfig config = DocumentCategorizerConfig.fromJson(json);
174+
assertNotNull(config);
175+
assertEquals(5, config.id2label().size());
176+
assertEquals("1 star", config.id2label().get("a0"));
177+
assertEquals("2 stars", config.id2label().get("a1"));
178+
assertEquals("3 stars", config.id2label().get("a2"));
179+
assertEquals("4 stars", config.id2label().get("a3"));
180+
assertEquals("5 stars", config.id2label().get("a4"));
181+
}
182+
}

0 commit comments

Comments
 (0)