Merge branch 'sample-deletion'

* sample-deletion:
  Upgraded PostgREST API
  Created a dedicated authenticator role
  Adjusted database migrations
  Sample deletion tests pass with PostgREST
  Fixed small issues with CLI configuration
  Sample deletion with security

_cli/sparrow_cli/commands/up.py

@@ -7,7 +7,6 @@
 
 from sparrow_cli.help import echo_messages
 
-from ..config.environment import validate_environment
 from ..config import SparrowConfig
 from ..util import compose, cmd, log
 from ..config.command_cache import get_backend_help_info
@@ -40,9 +39,9 @@ def sparrow_up(ctx, container="", force_recreate=False):
     # Validate the presence of SPARROW_SECREY_KEY only if we are bringing
     # the application up. Eventually, this should be wrapped into a Python
     # version of the `sparrow up` command.
-    validate_environment()
 
     cfg = ctx.find_object(SparrowConfig)
+    cfg.validate_environment()
 
     echo_messages(cfg)
 

_cli/sparrow_cli/config/__init__.py

@@ -6,7 +6,7 @@
 from os import environ
 from macrostrat.utils.shell import git_revision_info
 from macrostrat.utils.logs import get_logger, setup_stderr_logs
-from pydantic import BaseModel
+import hashlib
 import docker
 from docker.client import DockerClient
 from enum import Enum
@@ -133,6 +133,8 @@ def __init__(self, verbose=False, offline=False):
                 details=f"SPARROW_PATH={environ['SPARROW_PATH']}",
             )
 
+        self.enhance_secret_key()
+
         self.project_name = self.infer_project_name()
         self.local_frontend = self.configure_local_frontend()
 
@@ -172,6 +174,30 @@ def __init__(self, verbose=False, offline=False):
         else:
             self.messages += prepare_compose_overrides(self)
 
+    def enhance_secret_key(self):
+        secret_key = environ.get("SPARROW_SECRET_KEY")
+        if secret_key is None:
+            return
+        if len(secret_key) < 32:
+            self.add_message(
+                id="short-secret-key",
+                text="SPARROW_SECRET_KEY is shorter than 32 characters",
+                level=Level.WARNING,
+                details="This should be a randomly generated string of at least 32 characters.",
+            )
+            environ["SPARROW_SECRET_KEY"] = hashlib.md5(
+                secret_key.encode("utf-8")
+            ).hexdigest()
+
+    def validate_environment(self):
+        """Validate environment for services"""
+        # Ensure that the SPARROW_SECRET_KEY is set and >= 32 characters long
+        secret_key = environ.get("SPARROW_SECRET_KEY")
+        if secret_key is None:
+            raise SparrowCommandError(
+                "You must set the SPARROW_SECRET_KEY environment variable."
+            )
+
     def _setup_command_path(self):
         _bin = self.SPARROW_PATH / "_cli" / "bin"
         self.bin_directories = [_bin]
@@ -277,13 +303,19 @@ def configure_local_frontend(self):
             return False
         if not has_command("node"):
             self.add_message(
-                "Cannot run frontend locally without [bold]node[/bold] available.",
+                id="no-node",
+                text="Node is not installed",
+                details=[
+                    "Cannot run frontend locally without [bold]node[/bold] available."
+                ],
                 level=Level.ERROR,
             )
             _local_frontend = False
         if self.is_frozen:
             self.add_message(
-                "Cannot run frontend locally in a frozen environment.",
+                id="frozen-env",
+                text="Frozen environment",
+                details=["Cannot run frontend locally in a frozen environment."],
                 level=Level.ERROR,
             )
             _local_frontend = False

_cli/sparrow_cli/config/environment.py

@@ -41,7 +41,8 @@ def prepare_docker_environment():
     # Have to get rid of random printing to stdout in order to not break
     # logging and container ID
     # https://github.com/docker/scan-cli-plugin/issues/149
-    environ.setdefault("DOCKER_SCAN_SUGGEST", "false")
+    # environ.setdefault("DOCKER_SCAN_SUGGEST", "false")
+    environ.setdefault("DOCKER_CLI_HINTS", "false")
 
 
 def prepare_compose_overrides(cfg) -> List[Message]:
@@ -156,12 +157,3 @@ def add_override(name):
     environ["COMPOSE_FILE"] = ":".join(str(c) for c in compose_files)
     log.info(f"Docker compose overrides: {compose_files}")
     return messages
-
-
-def validate_environment():
-    # Check for failing environment
-    if environ.get("SPARROW_SECRET_KEY") is None:
-        print(
-            "[red]You [underline]must[/underline] set [bold]SPARROW_SECRET_KEY[/bold]. Exiting..."
-        )
-        sys.exit(1)

backend/conftest.py

@@ -1,11 +1,13 @@
 from pytest import fixture
 from os import environ
 from starlette.testclient import TestClient
+from requests import Session as RequestsSession
 from sparrow.core.app import Sparrow
 from sparrow.core.context import _setup_context
 from macrostrat.database.utils import wait_for_database
 from sparrow.core.auth.create_user import _create_user
 from sparrow.tests.helpers.database import testing_database
+from sqlalchemy.orm import scoped_session
 from sqlalchemy.orm import Session
 from sqlalchemy import event
 import logging
@@ -71,6 +73,43 @@ def app(pytestconfig):
         yield _app
 
 
+def wait_for_pgrest_api(postgrest_url):
+    """Wait for the PostgREST API to be available"""
+    from requests import get
+    from time import sleep
+
+    while True:
+        try:
+            get(postgrest_url)
+        except Exception as err:
+            print(err)
+            sleep(0.5)
+        else:
+            break
+
+
+@fixture(scope="session")
+def pg_api():
+    """PostgREST API client for testing"""
+    postgrest_url = environ.get("SPARROW_POSTGREST_URL")
+
+    if postgrest_url is None:
+        raise ValueError("SPARROW_POSTGREST_URL not set")
+
+    wait_for_pgrest_api(postgrest_url)
+
+    class PostgRESTSession(RequestsSession):
+        def request(self, method, url, *args, **kwargs):
+            if url.startswith("/"):
+                url = postgrest_url + url
+            return super().request(method, url, *args, **kwargs)
+
+    session = PostgRESTSession()
+    session.headers.update({"Accept": "application/json"})
+    yield session
+    session.close()
+
+
 @fixture(scope="function")
 def statements(db):
     stmts = []
@@ -84,10 +123,14 @@ def catch_queries(conn, cursor, statement, parameters, context, executemany):
 
 
 @fixture(scope="class")
-def db(app, pytestconfig):
+def db(app, pytestconfig, request):
     # https://docs.sqlalchemy.org/en/13/orm/session_transaction.html
     # https://gist.github.com/zzzeek/8443477
-    if pytestconfig.option.use_isolation:
+    use_isolation = pytestconfig.option.use_isolation
+    if request.cls is not None:
+        use_isolation = getattr(request.cls, "use_isolation", use_isolation)
+
+    if use_isolation:
         connection = app.database.engine.connect()
         transaction = connection.begin()
         session = Session(bind=connection)
@@ -114,6 +157,7 @@ def restart_savepoint(session, transaction):
         transaction.rollback()
         connection.close()
     else:
+        app.database.session = scoped_session(app.database._session_factory)
         yield app.database
 
 

backend/sparrow/core/auth/api.py

@@ -30,7 +30,9 @@ async def login(request, username: str, password: str):
 
     if current_user is not None and current_user.is_correct_password(password):
         day = 24 * 60 * 60
-        token = backend.set_cookie(None, "access", max_age=day, identity=username)
+        token = backend.set_cookie(
+            None, "access", max_age=day, identity=username, role="admin"
+        )
         resp = JSONResponse(dict(login=True, username=username, token=token))
         return backend.set_login_cookies(resp, identity=username)
 
@@ -59,7 +61,7 @@ def refresh(request):
     identity = backend.get_identity(request, type="refresh")
     response = JSONResponse(dict(login=True, refresh=True, username=identity))
 
-    return backend.set_access_cookie(response, identity=identity)
+    return backend.set_access_cookie(response, identity=identity, role="admin")
 
 
 @requires("admin")

backend/sparrow/core/auth/backend.py

@@ -6,6 +6,7 @@
 from typing import Tuple, Any
 
 import jwt
+import warnings
 from starlette.authentication import (
     BaseUser,
     SimpleUser,
@@ -83,7 +84,14 @@ def get_identity(self, request: Request, type="access"):
                 if "Authorization" not in request.headers:
                     raise AuthenticationError(f"Could not find {name} on request")
                 else:
-                    value = self._decode(request.headers["Authorization"])
+                    header = request.headers["Authorization"]
+                    if header.startswith("Bearer "):
+                        header = header[7:]
+                    else:
+                        warnings.warn(
+                            "Authorization header did not start with 'Bearer '. This is invalid and deprecated."
+                        )
+                    value = self._decode(header)
             else:
                 value = self._decode(cookie)
 

backend/sparrow/core/tags/fixtures/tags.sql

@@ -41,4 +41,11 @@ CREATE TABLE IF NOT EXISTS tags.project_tag (
     tag_id integer REFERENCES tags.tag(id) ON DELETE CASCADE,
     project_id integer REFERENCES project(id) ON DELETE CASCADE,
     PRIMARY KEY (tag_id, project_id)
-);
+);
+
+-- Add privileges
+GRANT USAGE ON SCHEMA tags TO view_public;
+GRANT SELECT ON ALL TABLES IN SCHEMA tags TO view_public;
+
+GRANT USAGE ON SCHEMA tags TO admin;
+GRANT SELECT, UPDATE, INSERT, DELETE ON ALL TABLES IN SCHEMA tags TO admin;

backend/sparrow/database/fixtures/05-security.sql

@@ -1,6 +1,19 @@
+-- Admin user can see all data in the database
 CREATE ROLE admin;
+
+GRANT USAGE ON SCHEMA public TO admin;
+GRANT USAGE ON SCHEMA vocabulary TO admin;
+GRANT SELECT, UPDATE, INSERT, DELETE ON ALL TABLES IN SCHEMA public TO admin;
+GRANT SELECT, UPDATE, INSERT, DELETE ON ALL TABLES IN SCHEMA vocabulary TO admin;
+
+-- Public user can see only public data
 CREATE ROLE view_public;
 
+GRANT USAGE ON SCHEMA public TO view_public;
+GRANT USAGE ON SCHEMA vocabulary TO view_public;
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO view_public;
+GRANT SELECT ON ALL TABLES IN SCHEMA vocabulary TO view_public;
+
 -- Lock down core tables
 
 ALTER TABLE datum ENABLE ROW LEVEL SECURITY;
@@ -74,17 +87,6 @@ FOR SELECT
 TO view_public
 USING (is_public(project));
 
-/* Allow admin to do everything */
-GRANT ALL ON datum TO admin;
-GRANT ALL ON analysis TO admin;
-GRANT ALL ON session TO admin;
-GRANT ALL ON sample TO admin;
-GRANT ALL ON project TO admin;
-
-/* Allow public to select only */
-GRANT SELECT ON datum TO view_public;
-GRANT SELECT ON analysis TO view_public;
-GRANT SELECT ON session TO view_public;
-GRANT SELECT ON sample TO view_public;
-GRANT SELECT ON project TO view_public;
+
+
 

backend/sparrow/database/fixtures/06-views.sql

@@ -212,7 +212,7 @@ SELECT id, description, authority
 FROM vocabulary.material;
 
 CREATE VIEW core_view.sample AS
-SELECT DISTINCT ON (s.id)
+SELECT
   s.id,
   s.igsn,
   s.name,

backend/sparrow/database/fixtures/07-postgrest-api.sql

@@ -1,3 +1,9 @@
+-- API-specific roles
+-- https://postgrest.org/en/stable/tutorials/tut0.html
+CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER;
+
+GRANT admin TO authenticator;
+GRANT view_public TO authenticator;
 
 CREATE SCHEMA IF NOT EXISTS sparrow_api;
 
@@ -17,8 +23,10 @@ CREATE OR REPLACE VIEW sparrow_api.session AS
 SELECT * FROM core_view.session;
 
 CREATE OR REPLACE VIEW sparrow_api.sample AS
-SELECT * FROM core_view.sample;
+SELECT * FROM sample;
 
+CREATE OR REPLACE VIEW sparrow_api.sample_data AS
+SELECT * FROM core_view.sample;
 
 CREATE OR REPLACE VIEW core_view.sample_v3 AS
 SELECT
@@ -119,4 +127,13 @@ CREATE OR REPLACE FUNCTION sparrow_api.sample_tile(
   )
   SELECT ST_AsMVT(grouped_features)
   FROM grouped_features;
-$$ LANGUAGE sql IMMUTABLE;
+$$ LANGUAGE sql IMMUTABLE;
+
+
+GRANT USAGE ON SCHEMA sparrow_api TO view_public;
+GRANT SELECT ON ALL TABLES IN SCHEMA sparrow_api TO view_public;
+GRANT USAGE ON SCHEMA sparrow_api TO admin;
+GRANT SELECT, UPDATE, INSERT, DELETE ON ALL TABLES IN SCHEMA sparrow_api TO admin;
+
+
+NOTIFY pgrst, 'reload schema'

backend/sparrow/migrations/sql/add-sample-srid.sql

@@ -1,4 +1,5 @@
 DROP SCHEMA IF EXISTS core_view CASCADE;
+DROP SCHEMA IF EXISTS sparrow_api CASCADE;
 
 UPDATE sample
 SET "location" = ST_Transform("location", 4326)