2023-08-10 | MAIN --> PROD | DEV (cb1652e) --> STAGING

.github/workflows/terraform-apply-env.yml

@@ -22,6 +22,7 @@ jobs:
       TF_VAR_cf_user: ${{ secrets.CF_USERNAME }}
       TF_VAR_cf_password: ${{ secrets.CF_PASSWORD }}
       TF_VAR_new_relic_license_key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+      TF_VAR_pgrst_jwt_secret: ${{ secrets.PGRST_JWT_SECRET }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       TERRAFORM_PRE_RUN: |
           apt-get update

.github/workflows/terraform-plan-env.yml

@@ -17,6 +17,7 @@ jobs:
       TF_VAR_cf_user: ${{ secrets.CF_USERNAME }}
       TF_VAR_cf_password: ${{ secrets.CF_PASSWORD }}
       TF_VAR_new_relic_license_key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+      TF_VAR_pgrst_jwt_secret: ${{ secrets.PGRST_JWT_SECRET }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       TERRAFORM_PRE_RUN: |
           apt-get update

.github/workflows/testing-from-ghcr.yml

@@ -15,6 +15,7 @@ jobs:
       SECRET_KEY: ${{ secrets.SECRET_KEY }}
       ALLOWED_HOSTS: '0.0.0.0 127.0.0.1 localhost'
       DISABLE_AUTH: False
+      PGRST_JWT_SECRET: ${{ secrets.PGRST_JWT_SECRET }}
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3

backend/.profile

@@ -38,10 +38,13 @@ export NEW_RELIC_HOST="gov-collector.newrelic.com"
 # being 0.
 if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then
     echo 'Starting API schema deprecation' &&
-    python manage.py drop_deprecated_api_schemas_and_views &&
+    python manage.py drop_deprecated_api_schema_and_views &&
     echo 'Finished API schema deprecation' &&
+    echo 'Dropping API schema' &&
+	python manage.py drop_api_schema &&
+	echo 'Finished dropping API schema' &&
     echo 'Starting API schema creation' &&
-    python manage.py create_api_schemas &&
+    python manage.py create_api_schema &&
     echo 'Finished API schema creation' &&
     echo 'Starting migrate' &&
     python manage.py migrate &&

backend/Makefile

@@ -54,8 +54,10 @@ nctest:
 docker-first-run:
 	docker compose build
 	docker compose run web bash -c '\
+		python manage.py drop_deprecated_api_schema_and_views &&\
+		python manage.py drop_api_schema &&\
 		python manage.py makemigrations &&\
-		python manage.py create_api_schemas &&\
+		python manage.py create_api_schema &&\
 		python manage.py migrate &&\
 		python manage.py create_api_views\
 		'

backend/api/test_views.py

@@ -386,6 +386,9 @@ def test_valid_data_creates_SAC_and_Access(self):
         )
         self.user.profile.save()
 
+        # Create a user to check that association happens on Access creation:
+        catcdotcom = baker.make(User, email="c@c.com")
+
         response = self.client.post(
             ACCESS_AND_SUBMISSION_PATH, VALID_ACCESS_AND_SUBMISSION_DATA, format="json"
         )
@@ -399,15 +402,21 @@ def test_valid_data_creates_SAC_and_Access(self):
         certifying_auditor_contact_access = Access.objects.get(
             sac=sac, role="certifying_auditor_contact"
         )
-        editors = [acc.email for acc in Access.objects.filter(sac=sac, role="editor")]
+
+        editors = Access.objects.filter(sac=sac, role="editor")
+        editor_emails = [acc.email for acc in editors]
+        editor_users = [acc.user for acc in editors]
 
         self.assertEqual(sac.submitted_by, self.user)
-        self.assertTrue(self.user.email in editors)
-        self.assertTrue("c@c.com" in editors)
-        self.assertTrue("d@d.com" in editors)
+        self.assertTrue(self.user.email in editor_emails)
+        self.assertTrue("c@c.com" in editor_emails)
+        self.assertTrue("d@d.com" in editor_emails)
         self.assertEqual(certifying_auditee_contact_access.email, "a@a.com")
         self.assertEqual(certifying_auditor_contact_access.email, "b@b.com")
 
+        # Check that existing user was associated without having had to log in:
+        self.assertIn(catcdotcom, editor_users)
+
     def test_multiple_auditee_auditor_contacts(self):
         """A new SAC is created along with related Access instances"""
         # Add eligibility and Auditee Info data to profile

backend/audit/models.py

@@ -456,12 +456,34 @@ def _general_info_get(self, key):
         return None
 
 
+class AccessManager(models.Manager):
+    """Custom manager for Access."""
+
+    def create(self, **obj_data):
+        """
+        Check for existing users and add them at access creation time.
+        Not doing this would mean that users logged in at time of Access
+        instance creation would have to log out and in again to get the new
+        access.
+        """
+        if obj_data["email"]:
+            try:
+                acc_user = User.objects.get(email=obj_data["email"])
+            except User.DoesNotExist:
+                acc_user = None
+            if acc_user:
+                obj_data["user"] = acc_user
+        return super().create(**obj_data)
+
+
 class Access(models.Model):
     """
     Email addresses which have been granted access to SAC instances.
     An email address may be associated with a User ID if an FAC account exists.
     """
 
+    objects = AccessManager()
+
     ROLES = (
         ("certifying_auditee_contact", _("Auditee Certifying Official")),
         ("certifying_auditor_contact", _("Auditor Certifying Official")),

backend/dissemination/README.md

@@ -1,12 +1,95 @@
+# Deploying a new API
+
+An API in PostgREST needs a few things to happen. 
+
+1. A JWT secret needs to be loaded into the PostgREST environment.
+2. We need to tear down what was
+3. We need to stand it back up again
+
+# Creating a JWT secret
+
+The command
+
+```
+fac create_api_jwt <role> <passphrase>
+```
+
+creates a JWT secret. The passphrase must be [at least 32 characters long](https://postgrest.org/en/v10.2/tutorials/tut1.html#:~:text=32%20characters%20long.-,Note,-Unix%20tools%20can).
+
+For example:
+
+```
+fac create_api_jwt api_fac_gov mooE1Olp7u3xwgeDihtrjX14vbX9fH27
+```
+
+This will create the JWT
+
+```
+eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiYXBpX2ZhY19nb3ZfYW5vbiIsImNyZWF0ZWQiOiIyMDIzLTA3LTE0VDIxOjA0OjA2LjEyMDU0OCIsImV4cGlyZXMiOiIyMDI0LTAxLTE0VDIxOjA0OjA2LjEyMDUyMCIsImV4cCI6MTcwNTI2NjI0Nn0.cu2EVrP5X5u6uxVffeHLDNI24pfYyyICKD3wm1UtWts
+```
+
+This has three pieces:
+
+```
+header.payload.signature
+```
+
+**The data can be decoded without the passphrase.** So, a JWT token is not a way of *encrypting* data. Do not put any privileged information in a JWT.
+
+However, without the passphrase, the signature cannot be verified. PostgREST will not accept a JWT as valid that does not have a good signature. Therefore, it should be the case that only JWTs we create, with this tool, signed with a passphrase we know, can be accepted by our stack as valid.
+
+We pass `role` and `exp`, which are fields that PostgREST expect. We add two human-readable fields, `created` and `expires`. All tokens generated with this tool expire after 6 months, and must be refreshed at api.data.gov. If we don't refresh the token, api.data.gov (meaning api.fac.gov) will stop working.
+
+## Using the JWT token
+
+For symmetric use, that passphrase must be loaded into a GH Secret, and that secret deployed to our environments via Terraform. In this way, our PostgREST container knows how to verify the integrity of JWTs that are sent to us.
+
+Our JWT only lives at api.data.gov. We will put it in the `Authorization: Bearer <jwt>` header. In this way, only API requests that come through api.data.gov (meaning requests that go to api.fac.gov) will be executed by PostgREST. All other queries, from all other sources, will be rejected.
+
+It is important that the role you choose matches the role we expect for public queries. Our schemas are attached to the role `api_fac_gov`. 
+
+For example:
+
+```
+curl -X GET -H "Authorization: Bearer ${JWT}" "${API_FAC_URL}/general?limit=1" 
+```
+
+should return one item from the general view. API_FAC_URL might be `http://localhost:3000` in testing locally, or `https://api.fac.gov` when working live.
+
+### Limiting access
+
+We use the `X-Api-Roles` header from api.data.gov to determine some levels of access.
+
+https://api-umbrella.readthedocs.io/en/latest/admin/api-backends/http-headers.html
+
+the stored procedure
+
+```
+has_tribal_data_access()
+```
+
+checks this header, and if the correct role is present (`fac_gov_tribal_data_access`), we will accept the query as being privileged. The role has to be attached to the key by an administrator.
+
+## Standing up / tearing down
+
+With each deployment of the stack, we should tear down and stand up the entire API. 
+
+1. `fac drop_deprecated_schema_and_views` will tear down any deprecated APIs. Always run it.
+1. `fac drop_api_schema` will tear down the active schema and everything associated with it.
+2. `fac create_api_schema` will create roles and the schema.
+3. `fac create_api_views` will create the views on the data.
+
+With this sequence, we completely tear down old *and* current APIs, as well as associated roles. Then, we stand them up again, including all roles. This guarantees that every deploy is a complete, fresh instantiation of the API, and any changes that may have been made to views, functions, or privileges are caught.
+
+In other words: the API should always be stood up from a "blank slate" in the name of stateless deploys.
+
 # API versions
 
 When adding a new API version.
 
 1. Create a folder in api/dissemination for the version name. E.g. `v1_0_1`. 
-2. Create `db_views.sql` and `init_api_db.sql`. Make sure the schema used throughout is your new version number.
-3. Update `.profile` in `backend`. The variable `API_VERSIONS` should be a comma-separated list of version numbers.
-4. Update `docker-compose.yml` and `docker-compose-web.yml` to change the `PGRST_DB_SCHEMAS` key to reflect all the active schemas. 
+2. Copy the contents of an existing API as a starting point.
+3. Update `docker-compose.yml` and `docker-compose-web.yml` to change the `PGRST_DB_SCHEMAS` key to reflect all the active schemas. 
    1. ADD TO THE END OF THIS LIST. The first entry is the default. Only add to the front of the list if we are certain the schema should become the new default.
    2. This is likely true of TESTED patch version bumps (v1_0_0 to v1_0_1), and *maybe* minor version bumps (v1_0_0 to v1_1_0). MAJOR bumps require change management messaging.
-5. Update `APIViewTests` to make sure you're testing the right schema. (That file might want some love...)
-
+4. Update `APIViewTests` to make sure you're testing the right schema. (That file might want some love...)

backend/dissemination/api/api/drop.sql

@@ -0,0 +1,10 @@
+
+begin;
+
+DROP SCHEMA IF EXISTS api CASCADE; 
+
+commit;
+
+notify pgrst,
+       'reload schema';
+

backend/dissemination/api/api_v1_0_0_beta/base.sql

@@ -0,0 +1,29 @@
+DO
+$do$
+BEGIN
+   IF EXISTS (
+      SELECT FROM pg_catalog.pg_roles
+      WHERE  rolname = 'authenticator') THEN
+      RAISE NOTICE 'Role "authenticator" already exists. Skipping.';
+   ELSE
+      CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER;
+   END IF;
+END
+$do$;
+
+DO
+$do$
+BEGIN
+   IF EXISTS (
+      SELECT FROM pg_catalog.pg_roles
+      WHERE  rolname = 'api_fac_gov') THEN
+      RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.';
+   ELSE
+      CREATE ROLE api_fac_gov NOLOGIN;
+   END IF;
+END
+$do$;
+
+GRANT api_fac_gov TO authenticator;
+
+NOTIFY pgrst, 'reload schema';

backend/dissemination/api/api_v1_0_0_beta/create_functions.sql

@@ -0,0 +1,60 @@
+-- WARNING
+-- Under PostgreSQL 12, the functions below work.
+-- Under PostgreSQL 14, these will break.
+--
+-- Note the differences:
+--
+-- raise info 'Works under PostgreSQL 12';
+-- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true));
+-- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true));
+-- raise info 'Works under PostgreSQL 14';
+-- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic');
+-- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires');
+--
+-- To quote the work of Dav Pilkey, "remember this now."
+
+create or replace function getter(base text, item text) returns text 
+as $getter$
+begin 
+	return current_setting(concat(base, '.', item), true);
+end;
+$getter$ language plpgsql;
+
+create or replace function get_jwt_claim(item text) returns text 
+as $get_jwt_claim$
+begin 
+	return getter('request.jwt.claim', item);
+end;
+$get_jwt_claim$ language plpgsql;
+
+create or replace function get_header(item text) returns text
+as $get_header$
+begin 
+	raise info 'request.header % %', item, getter('request.header', item);
+	return getter('request.header', item);
+end;
+$get_header$ LANGUAGE plpgsql;
+
+-- https://api-umbrella.readthedocs.io/en/latest/admin/api-backends/http-headers.html
+-- I'd like to go to a model where we provide the API keys. 
+-- However, for now, we're going to look for a role attached to an api.data.gov account.
+-- These come in on `X-Api-Roles` as a comma-separated string.
+create or replace function has_tribal_data_access() returns boolean
+as $has_tribal_data_access$
+declare 
+    roles text;
+begin 
+	select get_header('x-api-roles') into roles;
+    return (roles like '%fac_gov_tribal_access%');
+end;
+$has_tribal_data_access$ LANGUAGE plpgsql;
+
+create or replace function has_public_data_access_only() returns boolean
+as $has_public_data_access_only$
+begin 
+    return not has_tribal_data_access();
+end;
+$has_public_data_access_only$ LANGUAGE plpgsql;
+
+
+NOTIFY pgrst, 'reload schema';

backend/dissemination/api/api_v1_0_0_beta/create_schema.sql

@@ -1,26 +1,7 @@
--- Cloned from Lindsay's code in data_distro
-
--- We are self managing our migrations here
--- The SQL is called and applied from data_distro/migrations
--- This is a point in time of sql run that is comparable with data distro models. The views in the api_v1_0_0-beta_views folder represent the current views for the API_V1_0_0-BETA.
--- As the models change later, we don't want
--- our migration to reference fields that don't exist yet or already exist.
-
 --This will be used by the postgrest API_V1_0_0-BETA
 
 begin;
 
--- These need to be if statements because the schema and rolls already exist when you run tests
-do
-$$
-begin
-    if not exists (select * from pg_catalog.pg_roles where rolname = 'anon') then
-        create role anon;
-    end if;
-end
-$$
-;
-
 do
 $$
 begin
@@ -35,21 +16,33 @@ begin
             grant select
         -- this includes views
         on tables
-        to anon;
+        to api_fac_gov;
 
         -- Grant access to sequences, if we have them
-        grant usage on schema api_v1_0_0_beta to anon;
-        grant select, usage on all sequences in schema api_v1_0_0_beta to anon;
+        grant usage on schema api_v1_0_0_beta to api_fac_gov;
+        grant select, usage on all sequences in schema api_v1_0_0_beta to api_fac_gov;
         alter default privileges
             in schema api_v1_0_0_beta
             grant select, usage
         on sequences
-        to anon;
+        to api_fac_gov;
     end if;
 end
 $$
 ;
 
+-- This is the description
+COMMENT ON SCHEMA api_v1_0_0_beta IS
+    'The FAC dissemation API version 1.0.0-beta.'
+;
+
+-- https://postgrest.org/en/stable/references/api/openapi.html
+-- This is the title
+COMMENT ON SCHEMA api_v1_0_0_beta IS
+$$v1.0.0-beta
+
+A RESTful API that serves data from the SF-SAC.$$;
+
 commit;
 
 notify pgrst,