Skip to content

Commit 7d40116

Browse files
rzo1rzo1
committed
OPENNLP-1633 - Remove dependency towards jackson-databind in opennlp-dl module
1 parent 74c7d52 commit 7d40116

File tree

6 files changed

+237
-56
lines changed

6 files changed

+237
-56
lines changed

‎NOTICE

-6
Original file line numberDiff line numberDiff line change
@@ -93,9 +93,3 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
9393
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
9494
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
9595
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
96-
97-
============================================================================
98-
99-
jackson-databind
100-
https://github.com/FasterXML/jackson-databind
101-
The Apache Software License, Version 2.0

‎opennlp-brat-annotator/pom.xml

-7
Original file line numberDiff line numberDiff line change
@@ -61,13 +61,6 @@
6161
<artifactId>jackson-databind</artifactId>
6262
<version>${jackson.version}</version>
6363
<scope>runtime</scope>
64-
<exclusions>
65-
<!-- Byte-Buddy became a dependency by accident - TODO remove it with update version > 2.17.0 -->
66-
<exclusion>
67-
<groupId>net.bytebuddy</groupId>
68-
<artifactId>byte-buddy</artifactId>
69-
</exclusion>
70-
</exclusions>
7164
</dependency>
7265

7366
<dependency>

‎opennlp-dl/pom.xml

-12
Original file line numberDiff line numberDiff line change
@@ -41,18 +41,6 @@
4141
<artifactId>onnxruntime</artifactId>
4242
<version>${onnxruntime.version}</version>
4343
</dependency>
44-
<dependency>
45-
<groupId>com.fasterxml.jackson.core</groupId>
46-
<artifactId>jackson-databind</artifactId>
47-
<version>${jackson.version}</version>
48-
<exclusions>
49-
<!-- Byte-Buddy became a dependency by accident - TODO remove it with update version > 2.17.0 -->
50-
<exclusion>
51-
<groupId>net.bytebuddy</groupId>
52-
<artifactId>byte-buddy</artifactId>
53-
</exclusion>
54-
</exclusions>
55-
</dependency>
5644
<dependency>
5745
<groupId>org.slf4j</groupId>
5846
<artifactId>slf4j-api</artifactId>

‎opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerConfig.java

+29-6
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,41 @@
1818
package opennlp.dl.doccat;
1919

2020
import java.util.Collections;
21+
import java.util.HashMap;
2122
import java.util.Map;
23+
import java.util.Objects;
24+
import java.util.regex.Matcher;
25+
import java.util.regex.Pattern;
2226

23-
public class DocumentCategorizerConfig {
27+
public record DocumentCategorizerConfig(Map<String, String> id2label) {
2428

25-
private Map<String, String> id2label;
29+
private static final Pattern ID_TO_LABEL_PATTERN =
30+
Pattern.compile("\"id2label\"\\s*:\\s*\\{(.*?)\\}", Pattern.DOTALL);
31+
private static final Pattern ENTRY_PATTERN =
32+
Pattern.compile("\"([^\"]+)\"\\s*:\\s*\"(.*?)\"");
2633

27-
public Map<String, String> getId2label() {
34+
@Override
35+
public Map<String, String> id2label() {
2836
return Collections.unmodifiableMap(id2label);
2937
}
3038

31-
public void setId2label(Map<String, String> id2label) {
32-
this.id2label = id2label;
33-
}
39+
public static DocumentCategorizerConfig fromJson(String json) {
40+
Objects.requireNonNull(json, "json must not be null");
41+
42+
final Map<String, String> id2label = new HashMap<>();
43+
final Matcher matcher = ID_TO_LABEL_PATTERN.matcher(json);
44+
45+
if (matcher.find()) {
46+
final String id2labelContent = matcher.group(1);
47+
final Matcher entryMatcher = ENTRY_PATTERN.matcher(id2labelContent);
3448

49+
while (entryMatcher.find()) {
50+
final String key = entryMatcher.group(1);
51+
final String value = entryMatcher.group(2);
52+
id2label.put(key, value);
53+
}
54+
}
55+
56+
return new DocumentCategorizerConfig(id2label);
57+
}
3558
}

‎opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java

+17-25
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,6 @@
3636
import ai.onnxruntime.OrtEnvironment;
3737
import ai.onnxruntime.OrtException;
3838
import ai.onnxruntime.OrtSession;
39-
import com.fasterxml.jackson.databind.DeserializationFeature;
40-
import com.fasterxml.jackson.databind.ObjectMapper;
4139
import org.slf4j.Logger;
4240
import org.slf4j.LoggerFactory;
4341

@@ -67,16 +65,15 @@ public class DocumentCategorizerDL extends AbstractDL implements DocumentCategor
6765
/**
6866
* Instantiates a {@link DocumentCategorizer document categorizer} using ONNX models.
6967
*
70-
* @param model The ONNX model file.
71-
* @param vocabulary The model file's vocabulary file.
72-
* @param categories The categories.
68+
* @param model The ONNX model file.
69+
* @param vocabulary The model file's vocabulary file.
70+
* @param categories The categories.
7371
* @param classificationScoringStrategy Implementation of {@link ClassificationScoringStrategy} used
7472
* to calculate the classification scores given the score of each
7573
* individual document part.
76-
* @param inferenceOptions {@link InferenceOptions} to control the inference.
77-
*
74+
* @param inferenceOptions {@link InferenceOptions} to control the inference.
7875
* @throws OrtException Thrown if the {@code model} cannot be loaded.
79-
* @throws IOException Thrown if errors occurred loading the {@code model} or {@code vocabulary}.
76+
* @throws IOException Thrown if errors occurred loading the {@code model} or {@code vocabulary}.
8077
*/
8178
public DocumentCategorizerDL(File model, File vocabulary, Map<Integer, String> categories,
8279
ClassificationScoringStrategy classificationScoringStrategy,
@@ -102,21 +99,21 @@ public DocumentCategorizerDL(File model, File vocabulary, Map<Integer, String> c
10299
/**
103100
* Instantiates a {@link DocumentCategorizer document categorizer} using ONNX models.
104101
*
105-
* @param model The ONNX model file.
106-
* @param vocabulary The model file's vocabulary file.
107-
* @param config The model's config file. The file will be used to determine the classification categories.
102+
* @param model The ONNX model file.
103+
* @param vocabulary The model file's vocabulary file.
104+
* @param config The model's config file. The file will be used to
105+
* determine the classification categories.
108106
* @param classificationScoringStrategy Implementation of {@link ClassificationScoringStrategy} used
109107
* to calculate the classification scores given the score of each
110108
* individual document part.
111-
* @param inferenceOptions {@link InferenceOptions} to control the inference.
112-
*
109+
* @param inferenceOptions {@link InferenceOptions} to control the inference.
113110
* @throws OrtException Thrown if the {@code model} cannot be loaded.
114-
* @throws IOException Thrown if errors occurred loading the {@code model} or {@code vocabulary}.
111+
* @throws IOException Thrown if errors occurred loading the {@code model} or {@code vocabulary}.
115112
*/
116113
public DocumentCategorizerDL(File model, File vocabulary, File config,
117114
ClassificationScoringStrategy classificationScoringStrategy,
118115
InferenceOptions inferenceOptions)
119-
throws IOException, OrtException {
116+
throws IOException, OrtException {
120117

121118
this.env = OrtEnvironment.getEnvironment();
122119

@@ -175,7 +172,7 @@ public double[] categorize(String[] strings) {
175172
logger.error("Unload to perform document classification inference", ex);
176173
}
177174

178-
return new double[]{};
175+
return new double[] {};
179176

180177
}
181178

@@ -315,6 +312,7 @@ private List<Tokens> tokenize(final String text) {
315312

316313
/**
317314
* Applies softmax to an array of values.
315+
*
318316
* @param input An array of values.
319317
* @return The output array.
320318
*/
@@ -346,18 +344,12 @@ private int maxIndex(double[] arr) {
346344
}
347345

348346
private Map<Integer, String> readCategoriesFromFile(File config) throws IOException {
349-
350-
final String json = new String(Files.readAllBytes(config.toPath()));
351-
352-
final ObjectMapper objectMapper = new ObjectMapper();
353-
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
354-
355347
final DocumentCategorizerConfig documentCategorizerConfig =
356-
objectMapper.readValue(json, DocumentCategorizerConfig.class);
348+
DocumentCategorizerConfig.fromJson(new String(Files.readAllBytes(config.toPath())));
357349

358350
final Map<Integer, String> categories = new HashMap<>();
359-
for (final String key : documentCategorizerConfig.getId2label().keySet()) {
360-
categories.put(Integer.valueOf(key), documentCategorizerConfig.getId2label().get(key));
351+
for (final String key : documentCategorizerConfig.id2label().keySet()) {
352+
categories.put(Integer.valueOf(key), documentCategorizerConfig.id2label().get(key));
361353
}
362354

363355
return categories;

‎opennlp-dl/src/test/java/opennlp/dl/doccat/DocumentCategorizerConfigTest.java

+191
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package opennlp.dl.doccat;
18+
19+
import org.junit.jupiter.api.Test;
20+
21+
import static org.junit.jupiter.api.Assertions.assertEquals;
22+
import static org.junit.jupiter.api.Assertions.assertNotNull;
23+
24+
25+
public class DocumentCategorizerConfigTest {
26+
27+
@Test
28+
public void testId2LabelsFromJsonPrettyValid() {
29+
final String json = """
30+
{
31+
"_num_labels": 5,
32+
"architectures": [
33+
"BertForSequenceClassification"
34+
],
35+
"attention_probs_dropout_prob": 0.1,
36+
"directionality": "bidi",
37+
"finetuning_task": "sentiment-analysis",
38+
"hidden_act": "gelu",
39+
"hidden_dropout_prob": 0.1,
40+
"hidden_size": 768,
41+
"id2label": {
42+
"0": "1 star",
43+
"1": "2 stars",
44+
"2": "3 stars",
45+
"3": "4 stars",
46+
"4": "5 stars"
47+
},
48+
"initializer_range": 0.02,
49+
"intermediate_size": 3072,
50+
"label2id": {
51+
"1 star": 0,
52+
"2 stars": 1,
53+
"3 stars": 2,
54+
"4 stars": 3,
55+
"5 stars": 4
56+
},
57+
"layer_norm_eps": 1e-12,
58+
"max_position_embeddings": 512,
59+
"model_type": "bert",
60+
"num_attention_heads": 12,
61+
"num_hidden_layers": 12,
62+
"output_past": true,
63+
"pad_token_id": 0,
64+
"pooler_fc_size": 768,
65+
"pooler_num_attention_heads": 12,
66+
"pooler_num_fc_layers": 3,
67+
"pooler_size_per_head": 128,
68+
"pooler_type": "first_token_transform",
69+
"type_vocab_size": 2,
70+
"vocab_size": 105879
71+
}
72+
""";
73+
74+
final DocumentCategorizerConfig config = DocumentCategorizerConfig.fromJson(json);
75+
assertNotNull(config);
76+
assertEquals(5, config.id2label().size());
77+
assertEquals("1 star", config.id2label().get("0"));
78+
assertEquals("2 stars", config.id2label().get("1"));
79+
assertEquals("3 stars", config.id2label().get("2"));
80+
assertEquals("4 stars", config.id2label().get("3"));
81+
assertEquals("5 stars", config.id2label().get("4"));
82+
}
83+
84+
@Test
85+
public void testId2LabelsFromJsonUglyValid() {
86+
final String json = """
87+
{"_num_labels":5,"architectures":["BertForSequenceClassification"],"attention_probs_
88+
dropout_prob":0.1,"directionality":"bidi","finetuning_task":"sentiment-analysis",
89+
"hidden_act":"gelu","hidden_dropout_prob":0.1,"hidden_size":768,"id2label":{"0":"1 star",
90+
"1":"2 stars","2":"3 stars","3":"4 stars","4":"5 stars"},"initializer_range":0.02,
91+
"intermediate_size":3072,"label2id":{"1 star":0,"2 stars":1,"3 stars":2,"4 stars":3,"5
92+
stars":4},"layer_norm_eps":1e-12,"max_position_embeddings":512,"model_type":"bert",
93+
"num_attention_heads":12,"num_hidden_layers":12,"output_past":true,"pad_token_id":0,"
94+
pooler_fc_size":768,"pooler_num_attention_heads":12,"pooler_num_fc_layers":3,
95+
"pooler_size_per_head":128,"pooler_type":"first_token_transform","type_vocab_size":2,
96+
"vocab_size":105879}
97+
""";
98+
99+
final DocumentCategorizerConfig config = DocumentCategorizerConfig.fromJson(json);
100+
assertNotNull(config);
101+
assertEquals(5, config.id2label().size());
102+
assertEquals("1 star", config.id2label().get("0"));
103+
assertEquals("2 stars", config.id2label().get("1"));
104+
assertEquals("3 stars", config.id2label().get("2"));
105+
assertEquals("4 stars", config.id2label().get("3"));
106+
assertEquals("5 stars", config.id2label().get("4"));
107+
}
108+
109+
@Test
110+
public void testId2LabelsFromJsonNoValues() {
111+
final String json = """
112+
{"_num_labels":5,"architectures":["BertForSequenceClassification"],"attention_probs
113+
_dropout_prob":0.1,"directionality":"bidi","finetuning_task":"sentiment-analysis",
114+
"hidden_act":"gelu","hidden_dropout_prob":0.1,"hidden_size":768,"layer_norm_eps":1e-12,
115+
"max_position_embeddings":512,"model_type":"bert",
116+
"num_attention_heads":12,"num_hidden_layers":12,"output_past":true,"pad_token_id":0,
117+
"pooler_fc_size":768,"pooler_num_attention_heads":12,"pooler_num_fc_layers":3,
118+
"pooler_size_per_head":128,"pooler_type":"first_token_transform","type_vocab_size":2,
119+
"vocab_size":105879}
120+
""";
121+
122+
final DocumentCategorizerConfig config = DocumentCategorizerConfig.fromJson(json);
123+
assertNotNull(config);
124+
assertEquals(0, config.id2label().size());
125+
}
126+
127+
@Test
128+
public void testId2LabelsFromJsonEmptyInput() {
129+
final String json = "";
130+
final DocumentCategorizerConfig config = DocumentCategorizerConfig.fromJson(json);
131+
assertNotNull(config);
132+
assertEquals(0, config.id2label().size());
133+
}
134+
135+
@Test
136+
public void testId2LabelsFromJsonPrettyIdIsNotANumberValid() {
137+
final String json = """
138+
{
139+
"_num_labels": 5,
140+
"architectures": [
141+
"BertForSequenceClassification"
142+
],
143+
"attention_probs_dropout_prob": 0.1,
144+
"directionality": "bidi",
145+
"finetuning_task": "sentiment-analysis",
146+
"hidden_act": "gelu",
147+
"hidden_dropout_prob": 0.1,
148+
"hidden_size": 768,
149+
"id2label": {
150+
"a0": "1 star",
151+
"a1": "2 stars",
152+
"a2": "3 stars",
153+
"a3": "4 stars",
154+
"a4": "5 stars"
155+
},
156+
"initializer_range": 0.02,
157+
"intermediate_size": 3072,
158+
"label2id": {
159+
"1 star": "a0",
160+
"2 stars": "a1",
161+
"3 stars": "a2",
162+
"4 stars": "a3",
163+
"5 stars": "a4"
164+
},
165+
"layer_norm_eps": 1e-12,
166+
"max_position_embeddings": 512,
167+
"model_type": "bert",
168+
"num_attention_heads": 12,
169+
"num_hidden_layers": 12,
170+
"output_past": true,
171+
"pad_token_id": 0,
172+
"pooler_fc_size": 768,
173+
"pooler_num_attention_heads": 12,
174+
"pooler_num_fc_layers": 3,
175+
"pooler_size_per_head": 128,
176+
"pooler_type": "first_token_transform",
177+
"type_vocab_size": 2,
178+
"vocab_size": 105879
179+
}
180+
""";
181+
182+
final DocumentCategorizerConfig config = DocumentCategorizerConfig.fromJson(json);
183+
assertNotNull(config);
184+
assertEquals(5, config.id2label().size());
185+
assertEquals("1 star", config.id2label().get("a0"));
186+
assertEquals("2 stars", config.id2label().get("a1"));
187+
assertEquals("3 stars", config.id2label().get("a2"));
188+
assertEquals("4 stars", config.id2label().get("a3"));
189+
assertEquals("5 stars", config.id2label().get("a4"));
190+
}
191+
}

0 commit comments

Comments
 (0)