Merge branch 'master' into fix/expand

.pre-commit-config.yaml

@@ -4,6 +4,7 @@ repos:
     hooks:
     -   id: detect-secrets
         args: ['--baseline', '.secrets.baseline']
+        exclude: poetry.lock
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.5.0
     hooks:
@@ -12,6 +13,6 @@ repos:
     -   id: no-commit-to-branch
         args: [--branch, develop, --branch, master, --pattern, release/.*]
 -   repo: https://github.com/psf/black
-    rev: 19.10b0
+    rev: stable
     hooks:
     -   id: black

.secinclude

@@ -1,6 +1,5 @@
 bin/*
 indexd/*
-requirements.txt
-setup.py
-test-requirements.txt
+poetry.lock
+pyproject.toml
 wsgi.py

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": null,
     "lines": null
   },
-  "generated_at": "2020-10-30T15:26:11Z",
+  "generated_at": "2020-12-17T14:53:23Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -62,7 +62,7 @@
       {
         "hashed_secret": "8c0fdc72bf4daf2a2338287334d0cbd221fa9ef4",
         "is_verified": false,
-        "line_number": 26,
+        "line_number": 28,
         "type": "Base64 High Entropy String"
       }
     ],

.travis.yml

@@ -13,10 +13,12 @@ jdk:
     - oraclejdk8
 
 install:
-    - "python setup.py install --force"
-    - "pip install -r test-requirements.txt"
+    - curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
+    - source $HOME/.poetry/env
+    - poetry install -vv
 
-script: "py.test -vv --cov=indexd --cov-report xml tests"
+
+script: pytest -vv --cov=indexd --cov-report xml tests
 
 after_script:
   - COVERALLS_REPO_TOKEN=$COVERALLS_TOKEN coveralls

Dockerfile

@@ -11,15 +11,6 @@ RUN apk update \
     && apk add linux-headers musl-dev gcc \
     && apk add curl bash git vim logrotate
 
-COPY . /$appname
-COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini
-COPY ./deployment/uwsgi/wsgi.py /$appname/wsgi.py
-COPY clear_prometheus_multiproc /$appname/clear_prometheus_multiproc
-WORKDIR /$appname
-
-RUN python -m pip install --upgrade pip \
-    && python -m pip install --upgrade setuptools \
-    && pip install -r requirements.txt --use-deprecated=legacy-resolver
 
 RUN mkdir -p /var/www/$appname \
     && mkdir -p /var/www/.cache/Python-Eggs/ \
@@ -31,9 +22,26 @@ RUN mkdir -p /var/www/$appname \
 
 EXPOSE 80
 
+# install poetry
+RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
+
+COPY . /$appname
+COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini
+COPY ./deployment/uwsgi/wsgi.py /$appname/wsgi.py
+COPY clear_prometheus_multiproc /$appname/clear_prometheus_multiproc
+WORKDIR /$appname
+
+# cache so that poetry install will run if these files change
+COPY poetry.lock pyproject.toml /$appname/
+
+# install Indexd and dependencies via poetry
+RUN source $HOME/.poetry/env \
+    && poetry config virtualenvs.create false \
+    && poetry install -vv --no-dev --no-interaction \
+    && poetry show -v
+
 RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/index/version_data.py \
-    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/index/version_data.py \
-    && python setup.py install
+    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/index/version_data.py
 
 WORKDIR /var/www/$appname
 

README.md

@@ -303,28 +303,28 @@ On Linux
 sudo apt-get install python-psycopg2
 ```
 
-To install the implementation, simply run:
+To install the implementation, assure you have poetry installed and simply run:
 
 ```bash
-python setup.py install
+poetry install
 ```
 
 To see how the automated tests (run in Travis CI) install Indexd, check out the `.travis.yml` file in the root directory of this repository.
 
 ## Installation with Docker
 
 ```bash
-docker build --build-arg https_proxy=http://cloud-proxy:3128 --build-arg http_proxy=http://cloud-proxy:3128 -t Indexd .
+docker build --build-arg https_proxy=http://cloud-proxy:3128 --build-arg http_proxy=http://cloud-proxy:3128 -t indexd .
 
-docker run -d --name=Indexd -p 80:80 Indexd
-docker exec Indexd python /Indexd/bin/index_admin.py create --username $username --password $password
-docker exec Indexd python /Indexd/bin/index_admin.py delete --username $username
+docker run -d --name=indexd -p 80:80 Indexd
+docker exec indexd python /indexd/bin/index_admin.py create --username $username --password $password
+docker exec indexd python /indexd/bin/index_admin.py delete --username $username
 ```
 
 To run docker with an alternative settings file:
 
 ```
-docker run -d -v local_settings.py:/var/www/Indexd/local_settings.py --name=Indexd -p 80:80 Indexd
+docker run -d -v local_settings.py:/var/www/indexd/local_settings.py --name=Indexd -p 80:80 indexd
 ```
 
 ## Configuration
@@ -339,19 +339,9 @@ There is specific information about some configuration options in the [distribut
 - Follow [installation](#installation)
 - Run:
 ```
-pip install -r test-requirements.txt
-py.test -v tests/
+poetry run pytest tests/
 ```
 
-## Testing with Docker
-
-Doesn't work with all the DB tests yet, but you can adjust to run specific tests as necessary.
-
-```
-docker build -t Indexd -f TestDockerfile .
-```
-
-You should be able to use the test dockerfile to run some tests.
 
 <div align="center">
 <img src="./docs/gen3_large.png" alt="Gen3 Logo" height="425

bin/indexd

@@ -4,8 +4,6 @@ import sys
 import logging
 import argparse
 
-import flask
-
 from indexd import get_app
 
 
@@ -14,19 +12,12 @@ def setup_logging(log_levels, log_stream, **kwargs):
     Sets up basic logging.
     """
     logging.basicConfig(
-        level=min(log_levels), stream=log_stream,
+        level=min(log_levels),
+        stream=log_stream,
     )
 
 
-def main(host, port, debug_flask=False, **kwargs):
-    app = get_app()
-    app.run(
-        host=host, port=port, debug=debug_flask, threaded=True,
-    )
-
-
-if __name__ == "__main__":
-
+def parse_arguments():
     parser = argparse.ArgumentParser()
 
     parser.set_defaults(log_levels=[logging.ERROR])
@@ -57,25 +48,41 @@ if __name__ == "__main__":
     )
 
     parser.add_argument(
-        "--debug-flask", action="store_true", help="enable flask debugging",
+        "--debug-flask",
+        action="store_true",
+        help="enable flask debugging",
     )
 
     parser.add_argument(
-        "--host", default="localhost", help="host to server on [localhost]",
+        "--host",
+        default="localhost",
+        help="host to server on [localhost]",
     )
 
     parser.add_argument(
-        "--port", default=8080, type=int, help="port to server on [8080]",
+        "--port",
+        default=8080,
+        type=int,
+        help="port to server on [8080]",
     )
 
     args = parser.parse_args()
 
     setup_logging(**args.__dict__)
-
     logging.debug(args)
+    return args
 
+
+if __name__ == "__main__":
+    args = parse_arguments()
     try:
-        main(**args.__dict__)
+        app = get_app()
+        app.run(
+            host=args.host,
+            port=args.port,
+            debug=args.debug_flask,
+            threaded=True,
+        )
     except Exception as err:
         logging.exception(err)
         raise

bin/migrate_acl_authz.py

@@ -23,7 +23,6 @@
 
 import argparse
 import os
-import re
 import sys
 
 from cdislogging import get_logger

indexd/alias/drivers/alchemy.py

@@ -1,4 +1,3 @@
-import json
 import uuid
 
 from cdislogging import get_logger
@@ -17,7 +16,6 @@
 from sqlalchemy.orm.exc import MultipleResultsFound
 from sqlalchemy.ext.declarative import declarative_base
 
-from indexd import alias
 from indexd.alias.driver import AliasDriverABC
 
 from indexd.alias.errors import NoRecordFound

indexd/auth/drivers/alchemy.py

@@ -1,5 +1,3 @@
-import json
-import uuid
 import hashlib
 
 from contextlib import contextmanager
@@ -12,8 +10,6 @@
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.ext.declarative import declarative_base
 
-from indexd import auth
-
 from indexd.auth.driver import AuthDriverABC
 
 from indexd.auth.errors import AuthError

indexd/blueprint.py

@@ -1,6 +1,4 @@
-import re
 import flask
-import jsonschema
 
 from indexclient.client import IndexClient
 from doiclient.client import DOIClient

indexd/index/blueprint.py

@@ -470,7 +470,7 @@ def append_aliases(record):
     try:
         jsonschema.validate(aliases_json, RECORD_ALIAS_SCHEMA)
     except jsonschema.ValidationError as err:
-        logger.warn(f"Bad request body:\n{err}")
+        logger.warning(f"Bad request body:\n{err}")
         raise UserError(err)
 
     aliases = [record["value"] for record in aliases_json["aliases"]]
@@ -494,7 +494,7 @@ def replace_aliases(record):
     try:
         jsonschema.validate(aliases_json, RECORD_ALIAS_SCHEMA)
     except jsonschema.ValidationError as err:
-        logger.warn(f"Bad request body:\n{err}")
+        logger.warning(f"Bad request body:\n{err}")
         raise UserError(err)
 
     aliases = [record["value"] for record in aliases_json["aliases"]]
@@ -543,7 +543,7 @@ def update_all_index_record_versions(record):
     try:
         jsonschema.validate(request_json, UPDATE_ALL_VERSIONS_SCHEMA)
     except jsonschema.ValidationError as err:
-        logger.warn(f"Bad request body:\n{err}")
+        logger.warning(f"Bad request body:\n{err}")
         raise UserError(err)
 
     acl = request_json.get("acl")