Merge pull request #13 from UW-Macrostrat/docker-tests

Fix Docker containerization and tests

.env.example

@@ -1,4 +1,5 @@
 CORELLE_HTTP_PORT=5480
 CORELLE_HTTPS_PORT=5443
 CORELLE_DOMAIN=rotate.macrostrat.org
-COMPOSE_FILE=docker-compose.yaml:docker-compose.production.yaml
+COMPOSE_FILE=docker-compose.yaml:docker-compose.production.yaml
+POSTGRES_IMAGE=imresamu/postgis:15-3.4

Dockerfile

@@ -1,13 +1,13 @@
 # Dockerfile for the corelle API server
-FROM python:3.10
+FROM python:3.11
 
 WORKDIR /code
 
 ENV POETRY_VIRTUALENVS_CREATE=false
 
-RUN apt-get -y update && apt-get -y install postgresql-client
+RUN apt-get -y update && apt-get -y install postgresql-client gdal-bin libgdal-dev
 
-RUN pip install poetry==1.2.2
+RUN pip install poetry==1.7.1
 
 COPY ./poetry.lock ./pyproject.toml /code/
 COPY ./py-packages/client/pyproject.toml /code/py-packages/client/

Makefile

@@ -9,10 +9,16 @@ install:
 	make lock
 	poetry install
 
-init:
-	-createdb plate-rotations
-	poetry run corelle init --drop
-	poetry run bin/load-models
+test-docker:
+	bin/test-docker
+
+test:
+	poetry run bin/run-tests
+
+dev:
+	cd frontend && poetry run npm run dev
+
+# Outdated functions
 
 update_functions:
 	cat py-packages/engine/corelle/engine/schema/*-functions.sql | psql plate-rotations
@@ -22,8 +28,7 @@ baseurl := https://raw.githubusercontent.com/martynafford/natural-earth-geojson/
 features: bin/load-features
 	poetry run bin/load-features --redo plate-rotations
 
-test:
-	poetry run bin/run-tests
-
-dev:
-	cd frontend && poetry run npm run dev
+init:
+	-createdb plate-rotations
+	poetry run corelle init --drop
+	poetry run bin/load-models

bin/load-models

@@ -1,4 +1,5 @@
 #!/bin/sh
+# Note (2024-01-04): This script has been superseded by the `corelle import-starter-data` command
 
 if [ "$1" = "--redo" ]; then
   shift

bin/test-docker

@@ -1,17 +1,28 @@
 #!/usr/bin/env bash
+# This script is meant to be run from the root of the project
 
 set -e
 
+# Load dotenv if it exists
+if [ -f .env ]; then
+  set -o allexport
+  source .env
+  set +o allexport
+fi
+
+POSTGRES_IMAGE=${POSTGRES_IMAGE:-postgis/postgis:14-3.3}
+
 # Remove pre-existing containers if they exist
 docker rm -f corelle-db || true
 
-# Build the image
+# Build the image using buildkit
+export DOCKER_BUILDKIT=1
 docker build -t corelle .
 
 # Spin up the database
 docker run --name corelle-db -d \
   -e POSTGRES_HOST_AUTH_METHOD=trust \
-  -p 54321:5432 postgis/postgis:14-3.3
+  -p 54321:5432 $POSTGRES_IMAGE
 
 # Wait for the database to be ready
 until docker exec corelle-db pg_isready -h localhost -p 5432 -U postgres
@@ -29,11 +40,11 @@ docker run --rm \
   --volume $(pwd)/data:/code/data \
   -i corelle bash <<EOF
   corelle init
-  bin/load-models
+  corelle import-starter-data
 EOF
 
 # Run the tests
-docker run --rm --link corelle-db:database corelle /code/bin/run-tests
+docker run -t --rm --link corelle-db:database corelle /code/bin/run-tests $@
 
 # Stop the database
 docker stop corelle-db

py-packages/engine/corelle/engine/cache.py

@@ -16,6 +16,7 @@
 
 update_derived = get_sql("update-cache")
 
+
 def get_from_cache(cache_args):
     # Get a rotation from the database cache
     (model_name, plate_id, time) = cache_args
@@ -52,41 +53,47 @@ def build_rotation_cache(model, time_step=1):
     # Get model steps every 1 Myr
     t_steps = list(range(int(min_age), int(max_age) + 1, time_step))
 
-    with Progress() as progress:
-        task = progress.add_task(
-            f"Building cache for model {model.name}", total=len(t_steps)
-        )
-        rotations = get_rotation_series(model.name, *t_steps)
+    session = db.session()
+
+    session.execute(
+        "DELETE FROM corelle.rotation_cache WHERE model_id = :model_id",
+        dict(model_id=model.id),
+    )
+    session.commit()
+
+    rotations = get_rotation_series(model.name, *t_steps)
 
-        for tstep in rotations:
-            time = tstep["time"]
-            rows = [
-                build_cache_row(model.id, plate_id, time, q)
-                for plate_id, q in tstep["rotations"]
-                if q is not None
-            ]
-            add_to_cache(rows)
+    for tstep in rotations:
+        time = tstep["time"]
+        print(model.name, " - ", time, " Ma")
+        rows = [
+            build_cache_row(model.id, plate_id, time, q)
+            for plate_id, q in tstep["rotations"]
+            if q is not None
+        ]
+        add_to_cache(session, rows)
 
-            # Add derived columns
-            #conn.execute(update_derived, model_id=model.id, t_step=time)
-            progress.advance(task)
+        # Add derived columns
+        # session.execute(update_derived, model_id=model.id, t_step=time)
+        # progress.advance(task)
 
 
 def build_cache_row(model_id, plate_id, t_step, q):
-    lat, lon, angle = quaternion_to_euler(q)
-    # Create geometry as WKT
-    geom = f"SRID=4326;POINT({lon} {lat})"
+    # lat, lon, angle = quaternion_to_euler(q)
+    # # Create geometry as WKT
+    # geom = f"SRID=4326;POINT({lon} {lat})"
 
     return dict(
         model_id=int(model_id),
         plate_id=int(plate_id),
         t_step=float(t_step),
-        rotation=[q.w, q.x, q.y, q.z]
+        rotation=[q.w, q.x, q.y, q.z],
     )
 
 
-def add_to_cache(data):
+def add_to_cache(session, data):
     if not data:
         return
     # Add a rotation to the database cache
-    conn.execute(insert(_rotation_cache).on_conflict_do_nothing(), data)
+    session.execute(insert(_rotation_cache), data)
+    session.commit()

py-packages/engine/corelle/engine/cli.py

@@ -1,43 +1,33 @@
 import warnings
 
-import json
-import yaml
 import numpy as N
 from IPython import embed
-
-from click import group, argument, option, echo, Path
+from os import environ
+import click
+from click import group, argument, option, echo
 from corelle.math import quaternion_to_euler
-from os.path import splitext
+from macrostrat.utils import working_directory
+from pathlib import Path
 
 warnings.filterwarnings("ignore")
 
 
 @group()
 def cli():
+    """Register and pre-calculate plate-rotation models"""
     pass
 
 
 @cli.command(name="init")
 @option("--drop", is_flag=True, default=False)
 def init(drop=False):
+    """Create database fixtures"""
     from .database import initialize
 
     initialize(drop=drop)
 
 
-file = Path(exists=True, dir_okay=False)
-
-
-def load_fields(fn):
-    if not fn:
-        return None
-    ext = splitext(fn)[1]
-    with open(fn, "r") as f:
-        if ext == ".json":
-            return json.load(f)
-        if ext in [".yaml", ".yml"]:
-            return yaml.load(f, Loader=yaml.SafeLoader)
-    return None
+file = click.Path(exists=True, dir_okay=False)
 
 
 @cli.command(name="import")
@@ -62,7 +52,6 @@ def _import(
     """
     from .load_data import import_model
 
-    fields = load_fields(fields)
     import_model(
         model_name,
         plates,
@@ -87,6 +76,14 @@ def _import_features(name, file, overwrite=False):
     import_features(name, file, overwrite=False)
 
 
+@cli.command(name="import-starter-data")
+def import_basic():
+    """Import basic models and data"""
+    from .load_data import load_basic_data
+
+    load_basic_data()
+
+
 @cli.command(name="reset-cache")
 def cache():
     """
@@ -138,18 +135,41 @@ def rotate_all(model, time, verbose=False):
 @option("-p", "--port", type=int, default=5000)
 @option("--debug", is_flag=True, default=False)
 def serve(**kwargs):
+    """Run the application server"""
     from corelle.server import app
 
     app.run(host="0.0.0.0", **kwargs)
 
 
 @cli.command(name="shell")
 def shell():
+    """Get a shell in the application context"""
     embed()
 
 
 @cli.command(name="cache-rotations")
 def build_cache():
+    """Cache rotations for all models"""
     from .cache import build_rotation_caches
 
     build_rotation_caches()
+
+
+def parent_dir(_start: str, name: str) -> Path:
+    start = Path(_start)
+    while start.parent is not None:
+        start = start.parent
+        if start.name == name:
+            return start
+    return None
+
+
+@cli.command(name="test", context_settings=dict(ignore_unknown_options=True))
+def test(*args):
+    """Run tests"""
+    basedir = parent_dir(__file__, "py-packages")
+
+    with working_directory(basedir):
+        from pytest import main
+
+        main(list(args))

py-packages/engine/corelle/engine/database.py

@@ -1,7 +1,7 @@
 from os import environ
 from sqlalchemy.exc import ProgrammingError
 from macrostrat.utils import relative_path
-from macrostrat.database import Database
+from macrostrat.database import Database, run_sql
 from pathlib import Path
 
 conn_string = environ.get("CORELLE_DB", "postgresql:///plate-rotations")
@@ -20,4 +20,4 @@ def initialize(drop=False):
     files = sorted(dn.glob("*.sql"))
 
     for file in files:
-        list(db.run_sql(Path(dn) / file))
+        db.run_sql(Path(dn) / file, has_server_binds=False)