Adds job cancel methods, route and template UI

app/__init__.py

@@ -7,8 +7,10 @@
 from flask_migrate import Migrate
 
 from app.filters import usa_icon
-from app.scripts.load_manager import load_manager
 from database.models import db
+from harvester.lib.load_manager import LoadManager
+
+load_manager = LoadManager()
 
 load_dotenv()
 
@@ -35,7 +37,7 @@ def create_app():
 
     with app.app_context():
         db.create_all()
-        load_manager()
+        load_manager.start()
 
     return app
 

app/routes.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import os
 import secrets
@@ -14,13 +15,12 @@
 from flask import Blueprint, flash, redirect, render_template, request, session, url_for
 from jinja2_fragments.flask import render_block
 
-from app.scripts.load_manager import schedule_first_job, trigger_manual_job
 from database.interface import HarvesterDBInterface
+from harvester.lib.load_manager import LoadManager
 
 from . import htmx
 from .forms import HarvestSourceForm, OrganizationForm
 from .paginate import Pagination
-import json
 
 logger = logging.getLogger("harvest_admin")
 
@@ -32,6 +32,8 @@
 
 db = HarvesterDBInterface()
 
+load_manager = LoadManager()
+
 # Login authentication
 load_dotenv()
 CLIENT_ID = os.getenv("CLIENT_ID")
@@ -429,7 +431,7 @@ def add_harvest_source():
 
     if request.is_json:
         source = db.add_harvest_source(request.json)
-        job_message = schedule_first_job(source.id)
+        job_message = load_manager.schedule_first_job(source.id)
         if source and job_message:
             return {
                 "message": f"Added new harvest source with ID: {source.id}. {job_message}"
@@ -440,7 +442,7 @@ def add_harvest_source():
         if form.validate_on_submit():
             new_source = make_new_source_contract(form)
             source = db.add_harvest_source(new_source)
-            job_message = schedule_first_job(source.id)
+            job_message = load_manager.schedule_first_job(source.id)
             if source and job_message:
                 flash(f"Updated source with ID: {source.id}. {job_message}")
             else:
@@ -552,7 +554,7 @@ def edit_harvest_source(source_id: str):
             if form.validate_on_submit():
                 new_source_data = make_new_source_contract(form)
                 source = db.update_harvest_source(source_id, new_source_data)
-                job_message = schedule_first_job(source.id)
+                job_message = load_manager.schedule_first_job(source.id)
                 if source and job_message:
                     flash(f"Updated source with ID: {source.id}. {job_message}")
                 else:
@@ -618,7 +620,7 @@ def delete_harvest_source(source_id):
 ### Trigger Harvest
 @mod.route("/harvest_source/harvest/<source_id>", methods=["GET"])
 def trigger_harvest_source(source_id):
-    message = trigger_manual_job(source_id)
+    message = load_manager.trigger_manual_job(source_id)
     flash(message)
     return redirect(f"/harvest_source/{source_id}")
 
@@ -708,6 +710,15 @@ def delete_harvest_job(job_id):
     return result
 
 
+@mod.route("/harvest_job/cancel/<job_id>", methods=["GET", "POST"])
+@login_required
+def cancel_harvest_job(job_id):
+    """Cancels a harvest job"""
+    message = load_manager.stop_job(job_id)
+    flash(message)
+    return redirect(f"/harvest_job/{job_id}")
+
+
 ### Get Job Errors by Type
 @mod.route("/harvest_job/<job_id>/errors/<error_type>", methods=["GET"])
 def get_harvest_errors_by_job(job_id, error_type):
@@ -747,6 +758,7 @@ def get_harvest_records():
         records = db.pget_harvest_records(page)
     return db._to_dict(records)
 
+
 @mod.route("/harvest_record/<record_id>/raw", methods=["GET"])
 def get_harvest_record_raw(record_id=None):
     record = db.get_harvest_record(record_id)
@@ -759,6 +771,7 @@ def get_harvest_record_raw(record_id=None):
     else:
         return {"error": "Not Found"}, 404
 
+
 ### Add record
 @mod.route("/harvest_record/add", methods=["POST", "GET"])
 def add_harvest_record():

app/templates/view_job_data.html

@@ -27,7 +27,18 @@ <h2>Job Info</h2>
             {% endfor %}
         </table>
     </div>
-    <div class="section mb-3">
+    {% if session['user'] and data.harvest_job.status == "in_progress" %}
+    <div class="config-actions">
+        <ul class="usa-button-group">
+            <li class="usa-button-group__item">
+                <a href="{{ url_for('harvest.cancel_harvest_job', job_id=data.harvest_job_id)}}">
+                    <button class="usa-button usa-button--secondary">Cancel</button>
+                </a>
+            </li>
+        </ul>
+    </div>
+    {% endif %}
+    <div class="section my-3">
         <h2>Job Error Table</h2>
         {% if not data.harvest_job.errors %}
         No job errors found

app/templates/view_source_data.html

@@ -10,7 +10,8 @@
 {% endblock %}
 
 {% block content %}
-<div id="flash-message" style="display:none; position:fixed; top:10px; left:50%; transform:translateX(-50%); background-color:#ffcc00; color:black; padding:10px; border:1px solid #000; z-index:1000;">
+<div id="flash-message"
+    style="display:none; position:fixed; top:10px; left:50%; transform:translateX(-50%); background-color:#ffcc00; color:black; padding:10px; border:1px solid #000; z-index:1000;">
     Action in progress...
 </div>
 <div class="wrapper source-data">

harvester/harvest.py

@@ -473,6 +473,7 @@ def validate(self) -> None:
 
     def create_record(self, retry=False):
         from harvester.utils.ckan_utils import add_uuid_to_package_name
+
         try:
             result = ckan.action.package_create(**self.ckanified_metadata)
             self.ckan_id = result["id"]
@@ -513,10 +514,9 @@ def update_self_in_db(self) -> bool:
 
     def ckanify_dcatus(self) -> None:
         from harvester.utils.ckan_utils import ckanify_dcatus
+
         try:
-            self.ckanified_metadata = ckanify_dcatus(
-                self.metadata, self.harvest_source
-            )
+            self.ckanified_metadata = ckanify_dcatus(self.metadata, self.harvest_source)
         except Exception as e:
             self.status = "error"
             raise DCATUSToCKANException(