Merge pull request #9 from arthur-schnitzler/23-gnd-import-for-persons

23 gnd import for persons

.vscode/settings.json

@@ -1,3 +1,12 @@
 {
-    "python.formatting.provider": "black"
+    "python.formatting.provider": "black",
+    "cSpell.enableFiletypes": [
+        "!css",
+        "!python",
+        "!yaml",
+        "!yml",
+        "!html"
+    ],
+    "isort.check": true,
+    "isort.importStrategy": "fromEnvironment"
 }

apis_core/apis_entities/tests.py

@@ -5,6 +5,8 @@
 
 from apis_core.apis_entities.forms import get_entities_form
 from apis_core.apis_entities.models import Person
+from apis_core.apis_metainfo.models import Uri
+from normdata.forms import NormDataImportForm
 
 client = Client()
 USER = {"username": "testuser", "password": "somepassword"}
@@ -128,3 +130,45 @@ def test_010_delete_views(self):
                 self.assertContains(response, "Löschen von")
                 self.assertContains(response, item.id)
                 item.delete()
+
+    def test_010_import_nordmdata_view(self):
+        client.login(**USER)
+        payload = {
+            "normdata_url": "http://lobid.org/gnd/118566512",
+            "entity_type": "person",
+        }
+        url = reverse(
+            "normdata:import_from_normdata",
+        )
+        response = client.post(url, payload, follow=True)
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(Uri.objects.filter(uri__icontains="118566512"))
+        payload = {
+            "normdata_url": "https://www.geonames.org/2772400/linz.html",
+            "entity_type": "place",
+        }
+        response = client.post(url, payload, follow=True)
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(Uri.objects.filter(uri__icontains="2772400"))
+
+        payload = {
+            "normdata_url": "https://www.wikidata.org/wiki/Q119350694",
+            "entity_type": "person",
+        }
+        response = client.post(url, payload, follow=True)
+        self.assertEqual(response.status_code, 200)
+
+        payload = {
+            "normdata_url": "https://www.wikidata.org/wiki/Q119350694",
+            "entity_type": "person",
+        }
+        response = client.post(url, payload, follow=True)
+        self.assertEqual(response.status_code, 200)
+
+    def test_011_import_normdata_form(self):
+        payload = {
+            "normdata_url": "http://lobid.org/gnd/118566512",
+            "entity_type": "person"
+        }
+        form = NormDataImportForm(data=payload)
+        self.assertTrue(form.is_valid())

apis_core/apis_tei/tests.py

@@ -16,21 +16,24 @@
 
 
 class TeiTestCase(TestCase):
-
     def setUp(self):
         # Create two users
         User.objects.create_user(**USER)
         Person.objects.create(**BAHR)
 
     def test_01_tei_autocomplete(self):
-        url = reverse("apis:apis_tei:generic_entities_autocomplete", kwargs={"entity": "person"})
+        url = reverse(
+            "apis:apis_tei:generic_entities_autocomplete", kwargs={"entity": "person"}
+        )
         response = client.get(url)
         self.assertEqual(response.status_code, 200)
         response = client.get(f"{url}?q=Bahr")
         self.assertTrue("Hermann" in str(response.content))
 
     def test_02_tei_completer_autocomplete(self):
-        url = reverse("apis:apis_tei:tei_completer_autocomplete", kwargs={"entity": "person"})
+        url = reverse(
+            "apis:apis_tei:tei_completer_autocomplete", kwargs={"entity": "person"}
+        )
         response = client.get(url)
         self.assertEqual(response.status_code, 200)
         response = client.get(f"{url}?q=Bahr")

normdata/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class NormdataConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "normdata"

normdata/forms.py

@@ -0,0 +1,14 @@
+from django import forms
+
+
+class NormDataImportForm(forms.Form):
+    normdata_url = forms.URLField(
+        label="Normdata URL",
+        help_text="Zum Beispiel: http://lobid.org/gnd/118566512 oder https://www.geonames.org/2772400/linz.html",
+        max_length=100,
+    )
+    entity_type = forms.ChoiceField(
+        label="Entität",
+        help_text="Wähle die Art der Entität: Person, Ort, ...",
+        choices=(("person", "Person"), ("place", "Ort")),
+    )

normdata/templates/normdata/create_from_gnd.html

@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+{% load crispy_forms_tags %}
+{% load static %}
+{% block content %}
+<div class="container">
+    <h1 class="p-3 text-center">Neue Entität aus GND/WikiData/GeoNames importieren</h1>
+    <form action="." method="post">
+        {% csrf_token %}
+        {{ form|crispy }}
+        <input type="submit" value="Importieren">
+    </form>
+</div>
+
+{% endblock content %}

normdata/urls.py

@@ -0,0 +1,14 @@
+from django.urls import path
+
+from . import views
+
+app_name = "normdata"
+
+
+urlpatterns = [
+    path(
+        "import-from-normdata/",
+        views.NormDataImportFormView.as_view(),
+        name="import_from_normdata",
+    )
+]

normdata/utils.py

@@ -0,0 +1,87 @@
+from acdh_id_reconciler import geonames_to_wikidata, gnd_to_wikidata
+from acdh_wikidata_pyutils import WikiDataPerson, WikiDataPlace
+from AcdhArcheAssets.uri_norm_rules import get_normalized_uri
+from django.core.exceptions import ObjectDoesNotExist
+
+from apis_core.apis_entities.models import Person, Place
+from apis_core.apis_metainfo.models import Uri
+from dumper.utils import DOMAIN_MAPPING
+
+
+def get_uri_domain(uri):
+    for x in DOMAIN_MAPPING:
+        if x[0] in uri:
+            return x[1]
+
+
+def import_from_wikidata(wikidata_url, entity_type):
+    if entity_type == "person":
+        wd_entity = WikiDataPerson(wikidata_url)
+        apis_entity = wd_entity.get_apis_entity()
+        entity = Person.objects.create(**apis_entity)
+        Uri.objects.create(
+            uri=get_normalized_uri(wikidata_url),
+            domain="wikidata",
+            entity=entity,
+        )
+        if wd_entity.gnd_uri:
+            Uri.objects.create(
+                uri=get_normalized_uri(wd_entity.gnd_uri),
+                domain="gnd",
+                entity=entity,
+            )
+    else:
+        wd_entity = WikiDataPlace(wikidata_url)
+        apis_entity = wd_entity.get_apis_entity()
+        entity = Place.objects.create(**apis_entity)
+        Uri.objects.create(
+            uri=get_normalized_uri(wikidata_url),
+            domain="wikidata",
+            entity=entity,
+        )
+        if wd_entity.gnd_uri:
+            Uri.objects.create(
+                uri=get_normalized_uri(wd_entity.gnd_uri),
+                domain="gnd",
+                entity=entity,
+            )
+        if wd_entity.geonames_uri:
+            Uri.objects.create(
+                uri=get_normalized_uri(wd_entity.geonames_uri),
+                domain="geonames",
+                entity=entity,
+            )
+    return entity
+
+
+def import_from_normdata(raw_url, entity_type):
+    normalized_url = get_normalized_uri(raw_url)
+    try:
+        entity = Uri.objects.get(uri=normalized_url).entity
+        return entity
+    except ObjectDoesNotExist:
+        pass
+    domain = get_uri_domain(normalized_url)
+    if domain == "gnd":
+        try:
+            wikidata_url = gnd_to_wikidata(normalized_url)["wikidata"]
+        except (IndexError, KeyError):
+            wikidata_url = False
+    elif domain == "geonames":
+        try:
+            wikidata_url = geonames_to_wikidata(normalized_url)["wikidata"]
+        except (IndexError, KeyError):
+            wikidata_url = False
+    elif domain == "wikidata":
+        wikidata_url = normalized_url
+    else:
+        wikidata_url = False
+    if wikidata_url:
+        try:
+            entity = Uri.objects.get(uri=normalized_url).entity
+            return entity
+        except ObjectDoesNotExist:
+            entity = import_from_wikidata(wikidata_url, entity_type)
+    else:
+        entity = None
+    return entity

normdata/views.py

@@ -0,0 +1,19 @@
+from django.urls import reverse
+from django.views.generic.edit import FormView
+
+from .forms import NormDataImportForm
+from .utils import import_from_normdata
+
+
+class NormDataImportFormView(FormView):
+    template_name = "normdata/create_from_gnd.html"
+    form_class = NormDataImportForm
+
+    def get_success_url(self):
+        return reverse("apis:apis_entities:person_list_view")
+
+    def form_valid(self, form):
+        raw_url = form.data["normdata_url"]
+        entity_type = form.data["entity_type"]
+        import_from_normdata(raw_url, entity_type)
+        return super().form_valid(form)

pmb/settings.py

@@ -45,6 +45,7 @@
     "apis_core.apis_vocabularies",
     "apis_core.apis_labels",
     "apis_core.apis_tei",
+    "normdata",
     "dumper",
     "archemd",
 ]

pmb/urls.py

@@ -5,6 +5,7 @@
 
 urlpatterns = [
     path("apis/", include("apis_core.urls", namespace="apis")),
+    path("normdata/", include("normdata.urls", namespace="normdata")),
     path("admin/", admin.site.urls),
     path("arche/", include("archemd.urls")),
     path("", include("dumper.urls", namespace="dumper")),

requirements.txt

@@ -2,6 +2,7 @@ apis-override-select2js==0.1
 acdh-django-browsing
 acdh-id-reconciler>=0.2,<1
 acdh-tei-pyutils>=0.34,<1
+acdh-wikidata-pyutils>=0.2.1,<1
 Django>4.1,<6
 django-admin-csvexport
 django-autocomplete-light
@@ -11,6 +12,7 @@ django-model-utils
 django-tables2
 djangorestframework
 pandas
+pylobid
 psycopg2
 pyocclient==0.6
 icecream

templates/partials/navbar.html

@@ -70,6 +70,12 @@
                             </li>
                         </ul>
                     </li>
+                    {% if user.is_authenticated %}
+                    <li class="nav-item">
+                        <a class="nav-link" aria-current="page" href="{% url 'normdata:import_from_normdata' %}">Neue Entität aus GND/WikiData</a>
+                    </li>
+                    {% endif %}
+
                 </ul>
                 {% if user.is_authenticated %}
                 <a href="{% url 'dumper:user_logout' %}"><i class="bi bi-person-check"></i></a>