Add test data generation tool. (#217)

requirements/test.txt

@@ -1,5 +1,6 @@
 -r base.txt
 
+faker
 flake8==5.0.4
 pytest==5.4.3
 pytest_cov==2.11.1

tools/generate-test-data.py

@@ -0,0 +1,66 @@
+from datetime import date
+import sys
+
+sys.path.append("../AIPscan")
+
+from faker import Faker
+from flask import Flask
+from flask_sqlalchemy import SQLAlchemy
+
+from config import CONFIGS
+from AIPscan import db
+from AIPscan.models import AIP, FetchJob, File, StorageLocation, StorageService
+from helpers import data
+
+app = Flask(__name__)
+app.config.from_object(CONFIGS["default"])
+
+db.init_app(app)
+
+fake = Faker()
+randint = fake.random.randint
+
+with app.app_context():
+    # Add example storage services
+    ss_to_create = 2
+
+    print(f"Creating {ss_to_create} storage services...")
+
+    ss_ids = []
+    fetch_jobs = {}
+
+    default_created = False
+    for _ in range(ss_to_create):
+        is_default = len(ss_ids) == 0
+
+        ss = data.create_storage_service(is_default)
+        ss_ids.append(ss.id)
+
+        fetch_job = data.create_fetch_job(ss)
+        fetch_jobs[ss.id] = fetch_job.id
+
+    # Add example storage locations
+    storage_locations_per_ss = 2
+    ss_locations_to_create = ss_to_create * storage_locations_per_ss
+
+    print(f"Creating {ss_locations_to_create} storage service locations...")
+
+    aip_batches_created = 0
+    total_aip_batches = len(ss_ids) * storage_locations_per_ss
+    for ss_id in ss_ids:
+        for _ in range(storage_locations_per_ss):
+            sl = data.create_location(ss_id)
+
+            db.session.add(sl)
+            db.session.commit()
+
+            # Add AIPs
+            aip_batches_created += 1
+
+            print(f"Creating AIPs ({aip_batches_created}/{total_aip_batches})...")
+
+            for _ in range(1, randint(100, 300)):
+                aip = data.create_aip(ss_id, sl.id, fetch_jobs[ss.id])
+                data.create_aip_files(100, 300, aip)
+
+    print("Done.")

tools/helpers/data.py

@@ -0,0 +1,92 @@
+from datetime import date
+import sys
+
+sys.path.append("../AIPscan")
+
+from faker import Faker
+
+from config import CONFIGS
+from AIPscan import db
+from AIPscan.models import AIP, FetchJob, File, StorageLocation, StorageService
+
+fake = Faker()
+randint = fake.random.randint
+
+
+def create_storage_service(default):
+    ss = StorageService(
+        name=fake.text(20)[:-1],
+        url=fake.url(),
+        user_name=fake.profile()["username"],
+        api_key=fake.password(),
+        download_limit=0,
+        download_offset=0,
+        default=default,
+    )
+    db.session.add(ss)
+    db.session.commit()
+
+    return ss
+
+
+def create_fetch_job(storage_service):
+    fetch_job = FetchJob(
+        total_packages=0,
+        total_aips=0,
+        total_deleted_aips=0,
+        download_start=date.today(),
+        download_end=date.today(),
+        download_directory=fake.file_path(),
+        storage_service_id=storage_service.id,
+    )
+    db.session.add(fetch_job)
+    db.session.commit()
+
+    return fetch_job
+
+
+def create_location(storage_service_id):
+    return StorageLocation(
+        current_location=fake.file_path(),
+        description=fake.text(20)[:-1],
+        storage_service_id=storage_service_id,
+    )
+
+
+def create_aip(storage_service_id, storage_location_id, fetch_job_id):
+    aip = AIP(
+        uuid=fake.uuid4(),
+        transfer_name=fake.text(20)[:-1],
+        create_date=date.today(),
+        mets_sha256=fake.sha256(),
+        size=randint(10000, 100_000_000),
+        storage_service_id=storage_service_id,
+        storage_location_id=storage_location_id,
+        fetch_job_id=fetch_job_id,
+        origin_pipeline_id=1,
+    )
+    db.session.add(aip)
+    db.session.commit()
+
+    return aip
+
+
+def create_aip_files(min, max, aip):
+    for _ in range(1, randint(min, max)):
+        aipfile = File(
+            aip_id=aip.id,
+            name=fake.text(20)[:-1],
+            filepath=fake.file_path(),
+            uuid=fake.uuid4(),
+            file_type="original",
+            size=randint(1000, 1_000_000),
+            date_created=date.today(),
+            puid=fake.text(20)[:-1],
+            file_format=fake.text(20)[:-1],
+            format_version=fake.text(20)[:-1],
+            checksum_type=fake.text(20)[:-1],
+            checksum_value=fake.text(20)[:-1],
+            premis_object="",
+        )
+        db.session.add(aipfile)
+        db.session.commit()

tools/tests/test_data.py

@@ -0,0 +1,46 @@
+import sys
+
+import pytest
+
+from tools.helpers import data
+
+
+def test_create_storage_service(mocker):
+    mocker.patch("AIPscan.db.session.add")
+    mocker.patch("AIPscan.db.session.commit")
+
+    ss = data.create_storage_service(True)
+
+    assert len(ss.name) > 0
+    assert len(ss.url) > 0
+    assert len(ss.user_name) > 0
+    assert len(ss.api_key) > 0
+    assert ss.default == True
+
+    ss = data.create_storage_service(False)
+    assert ss.default == False
+
+
+def test_create_fetch_job(mocker):
+    mocker.patch("AIPscan.db.session.add")
+    mocker.patch("AIPscan.db.session.commit")
+
+    ss = data.create_storage_service(True)
+    ss.id = 1
+    fetch_job = data.create_fetch_job(ss)
+
+    assert len(str(fetch_job.download_start)) > 0
+    assert len(str(fetch_job.download_end)) > 0
+    assert len(fetch_job.download_directory) > 0
+    assert fetch_job.storage_service_id == ss.id
+
+
+def test_create_location(mocker):
+    mocker.patch("AIPscan.db.session.add")
+    mocker.patch("AIPscan.db.session.commit")
+
+    location = data.create_location(1)
+
+    assert len(location.current_location) > 0
+    assert len(location.description) > 0
+    assert location.storage_service_id == 1