Skip to content

Commit b455adb

Browse files
authored
Merge pull request #566 from dice-group/integrating-nero
Integrating NERO
2 parents 375b0e0 + e914010 commit b455adb

File tree

11 files changed

+1894
-86
lines changed

11 files changed

+1894
-86
lines changed

README.md

Lines changed: 36 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ including symbolic, neuro-symbolic, and deep learning-based approaches:
2626
- **NCES2** → [Neural Class Expression Synthesis in $\mathcal{ALCHIQ^{(D)}}$](https://papers.dice-research.org/2023/ECML_NCES2/NCES2_public.pdf)
2727
- **ROCES** → [Robust Class Expression Synthesis in Description Logics via Iterative Sampling](https://www.ijcai.org/proceedings/2024/0479.pdf)
2828
- **NCES** → [Neural Class Expression Synthesis](https://link.springer.com/chapter/10.1007/978-3-031-33455-9_13)
29-
- **NERO** → (soon) [Learning Permutation-Invariant Embeddings for Description Logic Concepts](https://link.springer.com/chapter/10.1007/978-3-031-30047-9_9)
29+
- **NERO** → [Learning Permutation-Invariant Embeddings for Description Logic Concepts](https://link.springer.com/chapter/10.1007/978-3-031-30047-9_9)
3030
- **CLIP** → [Learning Concept Lengths Accelerates Concept Learning in $\mathcal{ALC}$](https://link.springer.com/chapter/10.1007/978-3-031-06981-9_14)
3131
- **CELOE** → [Class Expression Learning for Ontology Engineering](https://www.sciencedirect.com/science/article/abs/pii/S1570826811000023)
3232
- **OCEL** → A limited version of CELOE
@@ -287,26 +287,26 @@ print(df[[col for col in df if col.startswith('Test-F1') or col.startswith('RT')
287287

288288
Below, we report the average test F1 score and the average runtimes of learners.
289289

290-
| LP | Test-F1-OCEL | RT-OCEL | Test-F1-CELOE | RT-CELOE | Test-F1-Evo | RT-Evo | Test-F1-DRILL | RT-DRILL | Test-F1-TDL | RT-TDL | Test-F1-NCES | RT-NCES | Test-F1-NCES2 | RT-NCES2 | Test-F1-ROCES | RT-ROCES | Test-F1-CLIP | RT-CLIP | Test-F1-ALCSAT | RT-ALCSAT | Test-F1-SPELL | RT-SPELL |
291-
|:------------------:|-------------:|--------:|--------------:|---------:|------------:|-------:|--------------:|---------:|------------:|-------:|-------------:|--------:|--------------:|---------:|--------------:|---------:|-------------:|--------:|----------------|-----------|---------------|----------|
292-
| Aunt | 0.614 | 13.697 | 0.855 | 13.697 | 0.978 | 5.278 | 0.811 | 60.351 | 0.956 | 0.118 | 0.805 | 0.632 | 0.812 | 1.136 | 0.812 | 1.119 | 0.855 | 14.059 | 1.000 | 0.618 | 0.690 | 4.121 |
293-
| Cousin | 0.712 | 10.846 | 0.789 | 10.846 | 0.993 | 3.311 | 0.701 | 60.485 | 0.820 | 0.176 | 0.608 | 0.628 | 0.680 | 1.177 | 0.695 | 1.086 | 0.779 | 9.050 | 1.000 | 0.749 | 0.715 | 5.346 |
294-
| Grandgranddaughter | 1.000 | 0.013 | 1.000 | 0.013 | 1.000 | 0.426 | 0.980 | 17.486 | 1.000 | 0.050 | 1.000 | 0.507 | 1.000 | 0.955 | 1.000 | 0.917 | 1.000 | 0.639 | 1.000 | 0.081 | 1.000 | > 0.001 |
295-
| Grandgrandfather | 1.000 | 0.897 | 1.000 | 0.897 | 1.000 | 0.404 | 0.947 | 55.728 | 0.947 | 0.059 | 0.927 | 0.505 | 0.947 | 0.944 | 0.927 | 0.924 | 1.000 | 0.746 | 1.000 | 0.144 | 1.000 | 0.002 |
296-
| Grandgrandmother | 1.000 | 4.173 | 1.000 | 4.173 | 1.000 | 0.442 | 0.893 | 50.329 | 0.947 | 0.060 | 0.947 | 0.633 | 0.933 | 1.323 | 0.947 | 1.306 | 1.000 | 0.817 | 0.980 | 0.141 | 0.980 | 0.003 |
297-
| Grandgrandson | 1.000 | 1.632 | 1.000 | 1.632 | 1.000 | 0.452 | 0.931 | 60.358 | 0.911 | 0.070 | 0.909 | 0.598 | 0.931 | 1.171 | 0.931 | 1.146 | 1.000 | 0.939 | 1.000 | 0.171 | 1.000 | 0.003 |
298-
| Uncle | 0.876 | 16.244 | 0.891 | 16.244 | 0.964 | 4.516 | 0.876 | 60.416 | 0.933 | 0.098 | 0.854 | 0.538 | 0.891 | 0.948 | 0.891 | 0.905 | 0.928 | 17.682 | 0.985 | 0.609 | 0.890 | 3.297 |
290+
| LP | Test-F1-OCEL | RT-OCEL | Test-F1-CELOE | RT-CELOE | Test-F1-Evo | RT-Evo | Test-F1-DRILL | RT-DRILL | Test-F1-TDL | RT-TDL | Test-F1-NCES | RT-NCES | Test-F1-NCES2 | RT-NCES2 | Test-F1-ROCES | RT-ROCES | Test-F1-CLIP | RT-CLIP | Test-F1-NERO | RT-NERO | Test-F1-ALCSAT | RT-ALCSAT | Test-F1-SPELL | RT-SPELL |
291+
|:------------------:|-------------:|--------:|--------------:|---------:|------------:|-------:|--------------:|---------:|------------:|-------:|-------------:|--------:|--------------:|---------:|--------------:|---------:|-------------:|--------:|--------------|---------|----------------|-----------|---------------|----------|
292+
| Aunt | 0.614 | 13.697 | 0.855 | 13.697 | 0.978 | 5.278 | 0.811 | 60.351 | 0.956 | 0.118 | 0.805 | 0.632 | 0.812 | 1.136 | 0.812 | 1.119 | 0.855 | 14.059 | 0.834 | 0.095 | 1.000 | 0.618 | 0.690 | 4.121 |
293+
| Cousin | 0.712 | 10.846 | 0.789 | 10.846 | 0.993 | 3.311 | 0.701 | 60.485 | 0.820 | 0.176 | 0.608 | 0.628 | 0.680 | 1.177 | 0.695 | 1.086 | 0.779 | 9.050 | 0.704 | 0.113 | 1.000 | 0.749 | 0.715 | 5.346 |
294+
| Grandgranddaughter | 1.000 | 0.013 | 1.000 | 0.013 | 1.000 | 0.426 | 0.980 | 17.486 | 1.000 | 0.050 | 1.000 | 0.507 | 1.000 | 0.955 | 1.000 | 0.917 | 1.000 | 0.639 | 1.000 | 0.092 | 1.000 | 0.081 | 1.000 | < 0.001 |
295+
| Grandgrandfather | 1.000 | 0.897 | 1.000 | 0.897 | 1.000 | 0.404 | 0.947 | 55.728 | 0.947 | 0.059 | 0.927 | 0.505 | 0.947 | 0.944 | 0.927 | 0.924 | 1.000 | 0.746 | 1.000 | 0.101 | 1.000 | 0.144 | 1.000 | 0.002 |
296+
| Grandgrandmother | 1.000 | 4.173 | 1.000 | 4.173 | 1.000 | 0.442 | 0.893 | 50.329 | 0.947 | 0.060 | 0.947 | 0.633 | 0.933 | 1.323 | 0.947 | 1.306 | 1.000 | 0.817 | 0.980 | 0.065 | 0.980 | 0.141 | 0.980 | 0.003 |
297+
| Grandgrandson | 1.000 | 1.632 | 1.000 | 1.632 | 1.000 | 0.452 | 0.931 | 60.358 | 0.911 | 0.070 | 0.909 | 0.598 | 0.931 | 1.171 | 0.931 | 1.146 | 1.000 | 0.939 | 0.980 | 0.095 | 1.000 | 0.171 | 1.000 | 0.003 |
298+
| Uncle | 0.876 | 16.244 | 0.891 | 16.244 | 0.964 | 4.516 | 0.876 | 60.416 | 0.933 | 0.098 | 0.854 | 0.538 | 0.891 | 0.948 | 0.891 | 0.905 | 0.928 | 17.682 | 0.881 | 0.098 | 0.985 | 0.609 | 0.890 | 3.297 |
299299

300300

301-
| LP | Train-F1-OCEL | Train-F1-CELOE | Train-F1-Evo | Train-F1-DRILL | Train-F1-TDL | Train-F1-NCES | Train-F1-NCES2 | Train-F1-ROCES | Train-F1-CLIP | Train-F1-ALCSAT | Train-F1-SPELL |
302-
|:------------------:|--------------:|---------------:|-------------:|---------------:|-------------:|----------------:|-----------------:|-----------------:|----------------:|-----------------|----------------|
303-
| Aunt | 0.835 | 0.918 | 0.995 | 0.837 | 1.000 | 0.759 | 0.804 | 0.804 | 0.918 | 1.000 | 0.756 |
304-
| Cousin | 0.746 | 0.796 | 1.000 | 0.732 | 1.000 | 0.680 | 0.696 | 0.728 | 0.798 | 1.000 | 0.720 |
305-
| Grandgranddaughter | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 |
306-
| Grandgrandfather | 1.000 | 1.000 | 1.000 | 0.968 | 1.000 | 0.910 | 0.944 | 0.942 | 1.000 | 1.000 | 1.000 |
307-
| Grandgrandmother | 1.000 | 1.000 | 1.000 | 0.975 | 1.000 | 0.923 | 0.941 | 0.944 | 1.000 | 1.000 | 1.000 |
308-
| Grandgrandson | 1.000 | 1.000 | 1.000 | 0.962 | 1.000 | 0.911 | 0.923 | 0.923 | 1.000 | 1.000 | 1.000 |
309-
| Uncle | 0.904 | 0.907 | 0.996 | 0.908 | 1.000 | 0.823 | 0.886 | 0.884 | 0.940 | 1.000 | 0.883 |
301+
| LP | Train-F1-OCEL | Train-F1-CELOE | Train-F1-Evo | Train-F1-DRILL | Train-F1-TDL | Train-F1-NCES | Train-F1-NCES2 | Train-F1-ROCES | Train-F1-CLIP | Train-F1-NERO | Train-F1-ALCSAT | Train-F1-SPELL |
302+
|:------------------:|--------------:|---------------:|-------------:|---------------:|-------------:|----------------:|-----------------:|-----------------:|----------------:|---------------|-----------------|----------------|
303+
| Aunt | 0.835 | 0.918 | 0.995 | 0.837 | 1.000 | 0.759 | 0.804 | 0.804 | 0.918 | 0.828 | 1.000 | 0.756 |
304+
| Cousin | 0.746 | 0.796 | 1.000 | 0.732 | 1.000 | 0.680 | 0.696 | 0.728 | 0.798 | 0.702 | 1.000 | 0.720 |
305+
| Grandgranddaughter | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 |
306+
| Grandgrandfather | 1.000 | 1.000 | 1.000 | 0.968 | 1.000 | 0.910 | 0.944 | 0.942 | 1.000 | 1.000 | 1.000 | 1.000 |
307+
| Grandgrandmother | 1.000 | 1.000 | 1.000 | 0.975 | 1.000 | 0.923 | 0.941 | 0.944 | 1.000 | 1.000 | 1.000 | 1.000 |
308+
| Grandgrandson | 1.000 | 1.000 | 1.000 | 0.962 | 1.000 | 0.911 | 0.923 | 0.923 | 1.000 | 0.979 | 1.000 | 1.000 |
309+
| Uncle | 0.904 | 0.907 | 0.996 | 0.908 | 1.000 | 0.823 | 0.886 | 0.884 | 0.940 | 0.884 | 1.000 | 0.883 |
310310

311311

312312
### 10-Fold Cross Validation Mutagenesis Benchmark Results
@@ -461,6 +461,23 @@ address="Cham"
461461
year={2022},
462462
publisher={Springer Nature Switzerland}
463463
}
464+
465+
# NERO
466+
@InProceedings{10.1007/978-3-031-30047-9_9,
467+
author="Demir, Caglar
468+
and Ngonga Ngomo, Axel-Cyrille",
469+
editor="Cr{\'e}milleux, Bruno
470+
and Hess, Sibylle
471+
and Nijssen, Siegfried",
472+
title="Learning Permutation-Invariant Embeddings for Description Logic Concepts",
473+
booktitle="Advances in Intelligent Data Analysis XXI",
474+
year="2023",
475+
publisher="Springer Nature Switzerland",
476+
address="Cham",
477+
pages="103--115",
478+
isbn="978-3-031-30047-9"
479+
}
480+
464481
```
465482

466483
In case you have any question or feedback, please contact us: ```caglar.demir@upb.de``` or ```alkid.baci@upb.de```.

docs/usage/09_further_resources.md

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,19 @@ You can find more details in the related papers for each algorithm:
44

55
Concept Learning:
66

7-
- **TDL** &rarr; Tree-based OWL Class Expression Learner for Large Graphs (manuscript will be added soon)
7+
- **TDL** &rarr; [Tree-based OWL Class Expression Learner for Large Graphs](https://dl.acm.org/doi/10.1007/978-3-032-06066-2_29)
88
- **Drill** &rarr; [Neuro-Symbolic Class Expression Learning](https://www.ijcai.org/proceedings/2023/0403.pdf)
99
- **EvoLearner** &rarr; [EvoLearner: Learning Description Logics with Evolutionary Algorithms](https://dl.acm.org/doi/abs/10.1145/3485447.3511925)
1010
- **NCES2** &rarr; [Neural Class Expression Synthesis in ALCHIQ(D)](https://papers.dice-research.org/2023/ECML_NCES2/NCES2_public.pdf)
1111
- **ROCES** &rarr; [Robust Class Expression Synthesis in Description Logics via Iterative Sampling](https://www.ijcai.org/proceedings/2024/0479.pdf)
1212
- **NCES** &rarr; [Neural Class Expression Synthesis](https://link.springer.com/chapter/10.1007/978-3-031-33455-9_13)
13-
- **NERO*** &rarr; (soon) [Learning Permutation-Invariant Embeddings for Description Logic Concepts](https://link.springer.com/chapter/10.1007/978-3-031-30047-9_9)
13+
- **NERO** &rarr; [Learning Permutation-Invariant Embeddings for Description Logic Concepts](https://link.springer.com/chapter/10.1007/978-3-031-30047-9_9)
1414
- **CLIP** &rarr; [Learning Concept Lengths Accelerates Concept Learning in ALC](https://link.springer.com/chapter/10.1007/978-3-031-06981-9_14)
1515
- **CELOE** &rarr; [Class Expression Learning for Ontology Engineering](https://www.sciencedirect.com/science/article/abs/pii/S1570826811000023)
1616
- **OCEL** &rarr; A limited version of CELOE
17+
- **SPELL** &rarr; [SAT-Based PAC Learning of Description Logic Concepts](https://www.ijcai.org/proceedings/2023/0373.pdf)
18+
- **ALCSAT** &rarr; [SAT-Based Bounded Fitting for the Description Logic $\mathcal{ALC}$](https://arxiv.org/pdf/2507.21752)
1719

18-
&ast; _Not implemented in our library yet._
1920

2021
Sampling:
2122
- **OntoSample** &rarr; [Accelerating Concept Learning via Sampling](https://dl.acm.org/doi/10.1145/3583780.3615158)
@@ -116,6 +117,34 @@ address="Cham"
116117
publisher={Springer Nature Switzerland}
117118
}
118119
120+
# NERO
121+
@InProceedings{10.1007/978-3-031-30047-9_9,
122+
author="Demir, Caglar
123+
and Ngonga Ngomo, Axel-Cyrille",
124+
editor="Cr{\'e}milleux, Bruno
125+
and Hess, Sibylle
126+
and Nijssen, Siegfried",
127+
title="Learning Permutation-Invariant Embeddings for Description Logic Concepts",
128+
booktitle="Advances in Intelligent Data Analysis XXI",
129+
year="2023",
130+
publisher="Springer Nature Switzerland",
131+
address="Cham",
132+
pages="103--115",
133+
abstract="Concept learning deals with learning description logic concepts from a background knowledge and input examples. The goal is to learn a concept that covers all positive examples, while not covering any negative examples. This non-trivial task is often formulated as a search problem within an infinite quasi-ordered concept space. Although state-of-the-art models have been successfully applied to tackle this problem, their large-scale applications have been severely hindered due to their excessive exploration incurring impractical runtimes. Here, we propose a remedy for this limitation. We reformulate the learning problem as a multi-label classification problem and propose a neural embedding model (NERO) that learns permutation-invariant embeddings for sets of examples tailored towards predicting {\$}{\$}F{\_}1{\$}{\$}F1scores of pre-selected description logic concepts. By ranking such concepts in descending order of predicted scores, a possible goal concept can be detected within few retrieval operations, i.e., no excessive exploration. Importantly, top-ranked concepts can be used to start the search procedure of state-of-the-art symbolic models in multiple advantageous regions of a concept space, rather than starting it in the most general concept {\$}{\$}{\backslash}top {\$}{\$}⊤. Our experiments on 5 benchmark datasets with 770 learning problems firmly suggest that NERO significantly (p-value {\$}{\$}<1{\backslash}{\%}{\$}{\$}<1{\%}) outperforms the state-of-the-art models in terms of {\$}{\$}F{\_}1{\$}{\$}F1score, the number of explored concepts, and the total runtime. We provide an open-source implementation of our approach (https://github.com/dice-group/Nero).",
134+
isbn="978-3-031-30047-9"
135+
}
136+
137+
# OWLAPY
138+
@misc{baci2025owlapypythonicframeworkowl,
139+
title={OWLAPY: A Pythonic Framework for OWL Ontology Engineering},
140+
author={Alkid Baci and Luke Friedrichs and Caglar Demir and Axel-Cyrille Ngonga Ngomo},
141+
year={2025},
142+
eprint={2511.08232},
143+
archivePrefix={arXiv},
144+
primaryClass={cs.SE},
145+
url={https://arxiv.org/abs/2511.08232},
146+
}
147+
119148
# OntoSample
120149
@inproceedings{10.1145/3583780.3615158,
121150
author = {Baci, Alkid and Heindorf, Stefan},

examples/concept_learning_cv_evaluation.py

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
from typing import Union
1212
import pandas as pd
1313
from ontolearn.knowledge_base import KnowledgeBase
14-
from ontolearn.learners import CELOE, EvoLearner, NCES, NCES2, ROCES, CLIP, ALCSAT, SPELL
14+
from ontolearn.learners import CELOE, EvoLearner, NCES, NCES2, ROCES, CLIP, ALCSAT, SPELL, NERO
1515
from ontolearn.refinement_operators import ModifiedCELOERefinement
1616
from ontolearn.learners import Drill, TDL, OCEL
1717
from ontolearn.learning_problem import PosNegLPStandard
@@ -127,6 +127,15 @@ def dl_concept_learning(args):
127127
max_query_size=10,
128128
search_mode="full_approx")
129129

130+
if not args.learner_types or 'nero' in args.learner_types:
131+
nero = NERO(knowledge_base=kb,
132+
num_embedding_dim=128,
133+
neural_architecture='DeepSet',
134+
learning_rate=0.001,
135+
num_epochs=50,
136+
batch_size=32
137+
)
138+
130139
# dictionary to store the data
131140
data = dict()
132141
if "problems" in settings:
@@ -427,6 +436,30 @@ def dl_concept_learning(args):
427436
print(f"SPELL Test Quality: {test_f1_spell:.3f}", end="\t")
428437
print(f"SPELL Runtime: {rt_spell:.3f}")
429438

439+
if not args.learner_types or 'nero' in args.learner_types:
440+
a_prop = list(kb.ontology.object_properties_in_signature())[:1].pop()
441+
ns = a_prop.iri.get_namespace()
442+
nero.ns = ns
443+
print("NERO starts..", end="\t")
444+
start_time = time.time()
445+
pred_nero = nero.fit(train_lp).best_hypothesis()
446+
rt_nero = time.time() - start_time
447+
print("NERO ends..", end="\t")
448+
# () Quality on the training data
449+
train_f1_nero = compute_f1_score(individuals=frozenset({i for i in kb.individuals(pred_nero)}),
450+
pos=train_lp.pos,
451+
neg=train_lp.neg)
452+
# () Quality on test data
453+
test_f1_nero = compute_f1_score(individuals=frozenset({i for i in kb.individuals(pred_nero)}),
454+
pos=test_lp.pos,
455+
neg=test_lp.neg)
456+
457+
data.setdefault("Train-F1-NERO", []).append(train_f1_nero)
458+
data.setdefault("Test-F1-NERO", []).append(test_f1_nero)
459+
data.setdefault("RT-NERO", []).append(rt_nero)
460+
print(f"NERO Train Quality: {train_f1_nero:.3f}", end="\t")
461+
print(f"NERO Test Quality: {test_f1_nero:.3f}", end="\t")
462+
print(f"NERO Runtime: {rt_nero:.3f}")
430463
df = pd.DataFrame.from_dict(data)
431464
df.to_csv(args.report, index=False)
432465
print(df)
@@ -441,7 +474,7 @@ def dl_concept_learning(args):
441474
parser.add_argument("--kb", type=str, required=True,
442475
help="Knowledge base")
443476
parser.add_argument("--learner_types", type=str, nargs='*', default=None,
444-
choices=["celoe", "ocel", "evolearner", "drill", "nces", "tdl", "nces2", "roces", "clip", "alcsat", "spell"],
477+
choices=["celoe", "ocel", "evolearner", "drill", "nces", "tdl", "nces2", "roces", "clip", "alcsat", "spell", "nero"],
445478
help="List of available concept learning models")
446479
parser.add_argument("--path_drill_embeddings", type=str, default=None)
447480
parser.add_argument("--path_of_nces_embeddings", type=str, default=None)

0 commit comments

Comments
 (0)