task/WG-268: add authentification to backend (#206)

* Rework env variables

* Add TAPIS_CLIENT_ID and  TAPIS_CLIENT_KEY
* Make env variable usage in docker compose consistent

* Add authentification flow

* Remove todo comment

* Add login endpoint test

* Add test for /auth/callback route

* Use access token and refresh if needed

* Use access token and refresh if needed

* Install mapillary tools by downloading first as using poetry causing errors on worker

Bumped python in worker image from 3.8 to 3.9 to match api image
Bumped other locked packages

* Fix worker image

* Rename public key used for validation

* Make mapillary_source relative for CI

* Bump poetry and mapillary_tools install

Update CI so that it works.

* Ensure we have auth for each user

* Commit new auth on existing users

* Rework Makefile so help i s first/default

* Update methods to check if auth exists

* Handle if no user has token in refresh_observable_projects

* Move tapis client instantiation after log statement

* Revert back to installing mapillary directly from github

* Add todo note

.github/workflows/main.yml

@@ -50,7 +50,7 @@ jobs:
     - name: Setup Poetry
       uses: abatilo/actions-poetry@v2.0.0
       with:
-        poetry-version: 1.4.2
+        poetry-version: 1.8.3
     - uses: actions/cache@v1
       with:
         path: ~/.cache/pypoetry/virtualenvs

Makefile

@@ -2,6 +2,12 @@ TAG := $(shell git log --format=%h -1)
 GEOAPI_IMAGE=taccaci/geoapi
 GEOAPI_WORKERS=taccaci/geoapi-workers
 
+
+.PHONY: help
+help:  ## Display this help screen
+	@grep -E '^([a-zA-Z_-]+):.*?## .*$$|^([a-zA-Z_-]+):' $(MAKEFILE_LIST) \
+	| awk 'BEGIN {FS = ":.*?## "}; {if ($$2) {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2} else {printf "\033[36m%-30s\033[0m %s\n", $$1, "(no description)"}}'
+
 .PHONY: start
 start:
 	docker-compose -f devops/docker-compose.local.yml --env-file .env up
@@ -44,8 +50,3 @@ deploy-geoapi:
 deploy-workers:
 	docker push $(GEOAPI_WORKERS):$(TAG)
 	docker push $(GEOAPI_WORKERS):latest
-
-.PHONY: help
-help:  ## Display this help screen
-	@grep -E '^([a-zA-Z_-]+):.*?## .*$$|^([a-zA-Z_-]+):' $(MAKEFILE_LIST) \
-	| awk 'BEGIN {FS = ":.*?## "}; {if ($$2) {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2} else {printf "\033[36m%-30s\033[0m %s\n", $$1, "(no description)"}}'

devops/Dockerfile

@@ -7,18 +7,17 @@ RUN apt-get update -q && apt-get install -q -y \
   curl \
   git
 
-ENV POETRY_VERSION=1.4.2
+ENV POETRY_VERSION=1.8.3
 ENV POETRY_HOME=/opt/poetry
 ENV PATH="$POETRY_HOME/bin:$PATH"
 RUN curl -sSL https://install.python-poetry.org | python3 -
 RUN poetry config virtualenvs.create false
 
-RUN mkdir /app
-WORKDIR /app
-
+WORKDIR /opt
 COPY devops/poetry.lock devops/pyproject.toml ./
 RUN poetry install
 
+RUN mkdir /app
 COPY geoapi /app/geoapi
 ENV PYTHONPATH "${PYTHONPATH}:/app"
 WORKDIR /app/geoapi

devops/Dockerfile.worker

@@ -6,7 +6,7 @@ RUN apt-get update && apt-get install -y \
 libtiff-dev libgeotiff-dev libgdal-dev \
 libboost-system-dev libboost-thread-dev libboost-filesystem-dev \
 libboost-program-options-dev libboost-regex-dev libboost-iostreams-dev \
-git cmake build-essential python3.8 python3-pip python3-dev ffmpeg \
+git cmake build-essential python3.9 python3-pip python3-dev ffmpeg \
 unzip git wget libc6-dev gcc-multilib
 
 WORKDIR /opt
@@ -20,24 +20,25 @@ cmake -DCMAKE_BUILD_TYPE=Release -DLASZIP_INCLUDE_DIRS=/opt/LAStools/LASzip/dll/
 make && make install && cp -r /opt/PotreeConverter/PotreeConverter/resources /resources
 ADD devops/misc/potree/page_template /resources/page_template
 
-RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 2
 
 RUN pip3 install --upgrade pip
 
-ENV POETRY_VERSION=1.4.2
+ENV POETRY_VERSION=1.8.3
 ENV POETRY_HOME=/opt/poetry
 ENV PATH="$POETRY_HOME/bin:$PATH"
 RUN curl -sSL https://install.python-poetry.org | python3 -
 RUN poetry config virtualenvs.create false
 COPY devops/pyproject.toml devops/poetry.lock ./
 
-RUN poetry install
-
-ENV PYTHONPATH "${PYTHONPATH}:/app"
+WORKDIR /opt
 
 # install geos into condo the base pdal image is using
-RUN conda install geos -y -n base
+RUN conda install setuptools geos -y -n base
 
+RUN poetry install
+
+ENV PYTHONPATH "${PYTHONPATH}:/app"
 
 WORKDIR /
 RUN mkdir app

devops/docker-compose.local.yml

@@ -49,11 +49,11 @@ services:
       - ../:/app
       - assets:/assets
     environment:
-      - MAPILLARY_CLIENT_TOKEN=${MAPILLARY_CLIENT_TOKEN}
+      - MAPILLARY_CLIENT_TOKEN
       - FLASK_APP=/app/geoapi/app.py
       - APP_ENV=local
       - ASSETS_BASE_DIR=/assets
-      - DESIGNSAFE_URL=${DESIGNSAFE_URL}
+      - DESIGNSAFE_URL
 
     stdin_open: true
     tty: true
@@ -92,11 +92,14 @@ services:
       - 8.8.8.8
       - 8.8.4.4
     environment:
-      - MAPILLARY_CLIENT_TOKEN=${MAPILLARY_CLIENT_TOKEN}
+      - MAPILLARY_CLIENT_TOKEN
       - FLASK_APP=/app/geoapi/app.py
       - APP_ENV=local
       - ASSETS_BASE_DIR=/assets
-      - DESIGNSAFE_URL=${DESIGNSAFE_URL}
+      - DESIGNSAFE_URL
+      - FLASK_SESSION_SECRET_KEY
+      - TAPIS_CLIENT_ID
+      - TAPIS_CLIENT_KEY
     stdin_open: true
     tty: true
     container_name: geoapi

devops/docker-compose.test.yml

@@ -60,6 +60,8 @@ services:
       - APP_ENV=testing
       - ASSETS_BASE_DIR=/assets
       - TENANT
+      - DESIGNSAFE_URL
+      - FLASK_SESSION_SECRET_KEY
     stdin_open: true
     tty: true
     networks:

devops/geoapi-services/docker-compose.yml

@@ -14,7 +14,6 @@ services:
     environment:
       - FLASK_APP=/app/geoapi/app.py
       - ASSETS_BASE_DIR=/assets
-      - GEOAPI_TAG=ENV_GEOAPI_TAG
     logging:
       driver: syslog
       options:
@@ -49,7 +48,6 @@ services:
     environment:
       - FLASK_APP=/app/geoapi/app.py
       - ASSETS_BASE_DIR=/assets
-      - GEOAPI_TAG=ENV_GEOAPI_TAG
     stdin_open: true
     tty: true
     container_name: geoapicelerybeat
@@ -74,8 +72,10 @@ services:
     environment:
       - FLASK_APP=/app/geoapi/app.py
       - ASSETS_BASE_DIR=/assets
-      - GEOAPI_TAG=ENV_GEOAPI_TAG
-      - DESIGNSAFE_URL=${DESIGNSAFE_URL}
+      - DESIGNSAFE_URL
+      - FLASK_SESSION_SECRET_KEY
+      - TAPIS_CLIENT_ID
+      - TAPIS_CLIENT_KEY
     stdin_open: true
     tty: true
     container_name: geoapi

devops/geoapi-services/secrets.sample.env

@@ -10,6 +10,11 @@ RABBITMQ_PASSWD='dev1234'
 RABBITMQ_DEFAULT_VHOST=dev
 RABBITMQ_VHOST=dev
 RABBITMQ_HOSTNAME=rabbitmq
-APP_ENV=development #  "production" for STAGING and PROD?
+APP_ENV=local # possible values are: production, staging, dev, experimental, local
 TENANT="{'DESIGNSAFE': {'service_account_token': 'ABCD1234'}}"
 MAPILLARY_CLIENT_TOKEN="ABC|1234564675757|abcd1234abcd12345"
+FLASK_SESSION_SECRET_KEY=abcd1234
+
+# Tapis clients
+TAPIS_CLIENT_ID=foo123
+TAPIS_CLIENT_KEY=bar123

devops/geoapi-workers/docker-compose.yml

@@ -9,8 +9,7 @@ services:
     environment:
       - FLASK_APP=/app/geoapi/app.py
       - ASSETS_BASE_DIR=/assets
-      - GEOAPI_TAG=ENV_GEOAPI_TAG
-      - DESIGNSAFE_URL=${DESIGNSAFE_URL}
+      - DESIGNSAFE_URL
     stdin_open: true
     tty: true
     container_name: geoapiworkers