fix(documents): delete orphan harvested documents

* Closes #3776.

Co-Authored-by: Pascal Repond <pascal.repond@rero.ch>

rero_ils/config.py

@@ -518,6 +518,14 @@ def _(x):
         "kwargs": {"delete": True},
         "enabled": False,
     },
+    "delete-orphan-harvested": {
+        "task": "rero_ils.modules.documents.tasks.delete_orphan_harvested",
+        "schedule": crontab(
+            minute=0, hour=5, day_of_week=6
+        ),  # Every Sunday at 05:00 UTC,
+        "kwargs": {"delete": True},
+        "enabled": False,
+    },
     # "mef-harvester": {
     #     "task": "rero_ils.modules.apiharvester.tasks.harvest_records",
     #     "schedule": timedelta(minutes=60),

rero_ils/modules/documents/tasks.py

@@ -36,6 +36,43 @@ def reindex_document(pid):
     Document.get_record_by_pid(pid).reindex()
 
 
+@shared_task(ignore_result=True)
+def delete_orphan_harvested(delete=False, verbose=False):
+    """Delete orphan harvested documents.
+
+    :param delete: if True delete from DB and ES.
+    :param verbose: Verbose print.
+    :returns: count of deleted documents.
+    """
+    query = (
+        DocumentsSearch()
+        .filter("term", harvested=True)
+        .exclude("exists", field="holdings")
+    )
+    pids = [hit.pid for hit in query.source("pid").scan()]
+    count = len(pids)
+
+    if verbose:
+        click.secho(f"Orphan harvested documents count: {count}", fg="yellow")
+    for pid in pids:
+        if doc := Document.get_record_by_pid(pid):
+            if verbose:
+                click.secho(f"Deleting orphan harvested: {pid}", fg="yellow")
+            if delete:
+                try:
+                    if doc.reasons_not_to_delete() == {"others": {"harvested": True}}:
+                        doc.delete(dbcommit=True, delindex=True, force=True)
+                except Exception:
+                    count -= 1
+                    msg = f"COULD NOT DELETE ORPHAN HARVESTED: {pid} {doc.reasons_not_to_delete()}"
+                    if verbose:
+                        click.secho(f"ERROR: {msg}", fg="red")
+                    current_app.logger.warning(msg)
+
+    set_timestamp("delete_orphan_harvested", msg={"deleted": count})
+    return count
+
+
 @shared_task(ignore_result=True)
 def delete_drafts(days=1, delete=False, verbose=False):
     """Delete drafts.

tests/ui/documents/test_documents_api.py

@@ -34,7 +34,7 @@
     document_id_fetcher,
 )
 from rero_ils.modules.documents.models import DocumentIdentifier
-from rero_ils.modules.documents.tasks import delete_drafts
+from rero_ils.modules.documents.tasks import delete_drafts, delete_orphan_harvested
 from rero_ils.modules.ebooks.tasks import create_records
 from rero_ils.modules.entities.models import EntityType
 from rero_ils.modules.entities.remote_entities.api import (
@@ -421,3 +421,17 @@ def test_document_delete_draft(app, document_chinese_data):
     doc["_draft"] = True
     doc.update(data=doc, dbcommit=True, reindex=True)
     assert delete_drafts(days=0, delete=True) == 1
+
+
+def test_document_delete_orphan_harvested(
+    app, document_data, holding_lib_sion_electronic
+):
+    """Test document delete orphan harvested.
+
+    Make sure that ebooks with electronic holdings (holding_lib_sion_electronic are not deleted).
+    """
+    doc = Document.create(data=document_data, dbcommit=True, reindex=True)
+    assert delete_orphan_harvested(delete=True) == 0
+    doc["harvested"] = True
+    doc.update(data=doc, dbcommit=True, reindex=True)
+    assert delete_orphan_harvested(delete=True) == 1