IGVF-55-add-linting-and-precommit-hooks (#14)

Author: ottojolanki

.circleci/config.yml

@@ -3,9 +3,21 @@ version: 2.1
 workflows:
   igvf-tests:
     jobs:
-      - run-tests
+      - lint
+      - run-tests:
+          requires:
+            - lint
 
 jobs:
+  lint:
+    docker:
+      - image: circleci/python:3.9
+    working_directory: ~/igvfd
+    steps:
+      - checkout
+      - run: pip install pre-commit==2.17.0
+      - run: pre-commit run --all-files
+
   run-tests:
     machine:
       image: ubuntu-2004:202111-02

.coveragerc

@@ -1,3 +1,3 @@
 [run]
-omit = 
-    */test*
+omit =
+    */test*

.dockerignore

@@ -1,2 +1,2 @@
 .test-reports
-.test-results
+.test-results

.gitignore

@@ -131,4 +131,4 @@ dmypy.json
 *~
 .DS_Store
 .test-reports
-.test-results
+.test-results

.pre-commit-config.yaml

@@ -0,0 +1,15 @@
+repos:
+  - repo: https://github.com/pre-commit/mirrors-autopep8
+    rev: v1.6.0
+    hooks:
+      - id: autopep8
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.2.3
+    hooks:
+      - id: double-quote-string-fixer
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: pretty-format-json
+        args: ['--autofix', '--no-sort-keys', '--indent=4']
+      - id: check-yaml

README.md

@@ -46,4 +46,12 @@ $ pytest
 4. Stop and clean.
 ```bash
 docker compose down -v
-```
+```
+
+## Automatic linting
+This repo includes configuration for pre-commit hooks. To use pre-commit, install pre-commit, and activate the hooks:
+```bash
+pip install pre-commit==2.17.0
+pre-commit install
+```
+Now every time you run `git commit` the automatic checks are run to check the changes you made.

config/docker_compose/postgres/Dockerfile

@@ -1 +1 @@
-FROM postgres:14.1
+FROM postgres:14.1

config/docker_compose/pyramid/Dockerfile

@@ -32,4 +32,4 @@ RUN pip install -e .[test]
 
 COPY . .
 
-ENTRYPOINT ["/docker/entrypoint.sh"]
+ENTRYPOINT ["/docker/entrypoint.sh"]

config/nginx/local.conf

@@ -41,4 +41,4 @@ http {
           proxy_pass $1$is_args$args;
       }
     }
-}
+}

config/pyramid/ini/docker.ini

@@ -52,7 +52,7 @@ level = NOTSET
 keys = console
 
 [logger_igvfd]
-handlers = 
+handlers =
 level = DEBUG
 qualname = igvfd
 
@@ -67,7 +67,7 @@ propagate = 0
 qualname = snovault.batchupgrade
 
 [logger_wsgi]
-handlers = 
+handlers =
 level = DEBUG
 qualname = wsgi
 
@@ -82,4 +82,4 @@ set pyramid.includes = pyramid_translogger
 host = 0.0.0.0
 port = 6543
 threads = 1
-use = egg:waitress#main
+use = egg:waitress#main

config/pyramid/ini/local.ini

@@ -51,7 +51,7 @@ level = NOTSET
 keys = console
 
 [logger_igvfd]
-handlers = 
+handlers =
 level = DEBUG
 qualname = igvfd
 
@@ -66,7 +66,7 @@ propagate = 0
 qualname = snovault.batchupgrade
 
 [logger_wsgi]
-handlers = 
+handlers =
 level = DEBUG
 qualname = wsgi
 
@@ -81,4 +81,4 @@ set pyramid.includes = pyramid_translogger
 host = 0.0.0.0
 port = 6543
 threads = 1
-use = egg:waitress#main
+use = egg:waitress#main

config/pyramid/ini/testing.ini

@@ -52,7 +52,7 @@ level = NOTSET
 keys = console
 
 [logger_igvfd]
-handlers = 
+handlers =
 level = DEBUG
 qualname = igvfd
 
@@ -67,7 +67,7 @@ propagate = 0
 qualname = snovault.batchupgrade
 
 [logger_wsgi]
-handlers = 
+handlers =
 level = DEBUG
 qualname = wsgi
 
@@ -82,4 +82,4 @@ set pyramid.includes = pyramid_translogger
 host = 0.0.0.0
 port = 6543
 threads = 1
-use = egg:waitress#main
+use = egg:waitress#main

docker-compose.test.yml

@@ -33,4 +33,4 @@ services:
     ports:
       - "8000:8000"
     depends_on:
-      - pyramid
+      - pyramid

docker-compose.yml

@@ -45,4 +45,4 @@ services:
     ports:
       - "8000:8000"
     depends_on:
-      - pyramid
+      - pyramid

pyproject.toml

@@ -1,6 +1,9 @@
+[tool.autopep8]
+max_line_length = 120
+
 [build-system]
 requires = [
     "setuptools>=42",
     "wheel"
 ]
-build-backend = "setuptools.build_meta"
+build-backend = "setuptools.build_meta"

setup.cfg

@@ -72,4 +72,4 @@ console_scripts =
 paste.app_factory =
     main = igvfd:main
 paste.filter_app_factory =
-    memlimit = igvfd.memlimit:filter_app
+    memlimit = igvfd.memlimit:filter_app

src/igvfd/__init__.py

@@ -1,7 +1,7 @@
 __version__ = '0.0.1'
 
 
-import igvfd.schema_formats # needed to import before snovault to add FormatCheckers
+import igvfd.schema_formats  # needed to import before snovault to add FormatCheckers
 import base64
 import codecs
 import copy
@@ -29,7 +29,7 @@
 
 
 def json_asset(spec, **kw):
-    utf8 = codecs.getreader("utf-8")
+    utf8 = codecs.getreader('utf-8')
     asset = AssetResolver(caller_package()).resolve(spec)
     return json.load(utf8(asset.stream()), **kw)
 
@@ -66,8 +66,8 @@ def changelogs(config):
 
 def configure_engine(settings):
     settings = copy.deepcopy(settings)
-    engine_url = os.environ.get("SQLALCHEMY_URL") or settings['sqlalchemy.url']
-    settings["sqlalchemy.url"] = engine_url
+    engine_url = os.environ.get('SQLALCHEMY_URL') or settings['sqlalchemy.url']
+    settings['sqlalchemy.url'] = engine_url
     engine_opts = {}
     if engine_url.startswith('postgresql'):
         application_name = 'app'
@@ -95,7 +95,7 @@ def set_postgresql_statement_timeout(engine, timeout=20 * 1000):
     def connect(dbapi_connection, connection_record):
         cursor = dbapi_connection.cursor()
         try:
-            cursor.execute("SET statement_timeout TO %d" % timeout)
+            cursor.execute('SET statement_timeout TO %d' % timeout)
         except psycopg2.Error:
             dbapi_connection.rollback()
         finally:

src/igvfd/auth0.py

@@ -46,6 +46,7 @@ def includeme(config):
 class LoginDenied(HTTPForbidden):
     title = 'Login failure'
 
+
 class Auth0AuthenticationPolicy(CallbackAuthenticationPolicy):
     """
     Checks assertion during authentication so login can construct user session.
@@ -72,10 +73,10 @@ def unauthenticated_userid(self, request):
                     request)
             request._auth0_authenticated = None
             return None
-        
+
         try:
             domain = 'encode.auth0.com'
-            user_url = "https://{domain}/userinfo?access_token={access_token}" \
+            user_url = 'https://{domain}/userinfo?access_token={access_token}' \
                 .format(domain=domain, access_token=access_token)
             user_info = requests.get(user_url).json()
         except Exception as e:
@@ -93,7 +94,6 @@ def unauthenticated_userid(self, request):
         else:
             return None
 
-
     def remember(self, request, principal, **kw):
         return []
 

src/igvfd/authentication.py

@@ -97,7 +97,7 @@ def basic_auth_check(username, password, request):
         return None
 
     #valid, new_hash = crypt_context.verify_and_update(password, hash)
-    #if new_hash:
+    # if new_hash:
     #    replace_user_hash(user, new_hash)
 
     return []

src/igvfd/commands/run_local.py

@@ -56,10 +56,10 @@ def nginx_server_process(prefix='', echo=False):
 
 
 def main():
-    set_start_method("fork")
+    set_start_method('fork')
     import argparse
     parser = argparse.ArgumentParser(
-        description="Run development servers", epilog=EPILOG,
+        description='Run development servers', epilog=EPILOG,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     parser.add_argument(
@@ -72,10 +72,10 @@ def main():
         default=f'{Path().absolute()}/config/pyramid/ini/local.ini',
         help='path to configfile',
     )
-    parser.add_argument('--clear', action="store_true", help="Clear existing data")
-    parser.add_argument('--init', action="store_true", help="Init database")
-    parser.add_argument('--load', action="store_true", help="Load test set")
-    parser.add_argument('--datadir', default='/tmp/igvfd', help="path to datadir")
+    parser.add_argument('--clear', action='store_true', help='Clear existing data')
+    parser.add_argument('--init', action='store_true', help='Init database')
+    parser.add_argument('--load', action='store_true', help='Load test set')
+    parser.add_argument('--datadir', default='/tmp/igvfd', help='path to datadir')
     args = parser.parse_args()
 
     logging.basicConfig()
@@ -132,5 +132,6 @@ def cleanup_process():
     for p in print_processes:
         p.start()
 
+
 if __name__ == '__main__':
     main()

src/igvfd/edw_hash.py

@@ -21,8 +21,8 @@ class EDWHash(uh.StaticHandler):
     checksum_size = 64
 
     setting_kwds = ('salt_before', 'salt_after', 'salt_base')
-    salt_before = b"186ED79BAEXzeusdioIsdklnw88e86cd73"
-    salt_after = b"<*#$*(#)!DSDFOUIHLjksdf"
+    salt_before = b'186ED79BAEXzeusdioIsdklnw88e86cd73'
+    salt_after = b'<*#$*(#)!DSDFOUIHLjksdf'
     salt_base = b64decode(b"""\
 Kf8r/S37L/kh9yP1JfMn8TnvO+096z/pMecz5TXjN+EJ3wvdDdsP2QHXA9UF0wfRGc8bzR3LH8kR
 xxPFFcMXwWm/a71tu2+5YbdjtWWzZ7F5r3utfat/qXGnc6V1o3ehSZ9LnU2bT5lBl0OVRZNHkVmP
@@ -40,7 +40,7 @@ def _calc_checksum(self, secret):
             secret = secret.encode('utf-8')
         salted = self.salt_before + secret + self.salt_after + b'\0'
         if len(salted) > len(self.salt_base):
-            raise ValueError("Password too long")
+            raise ValueError('Password too long')
         salted += self.salt_base[len(salted):]
         chk = sha384(salted).digest()
-        return b64encode(chk).decode("ascii")
+        return b64encode(chk).decode('ascii')

src/igvfd/loadxl.py

@@ -421,13 +421,13 @@ def attachment(path):
             im = Image.open(stream)
             im.verify()
             if im.format != minor.upper():
-                msg = "Image file format %r does not match extension for %s"
+                msg = 'Image file format %r does not match extension for %s'
                 raise ValueError(msg % (im.format, filename))
 
             attach['width'], attach['height'] = im.size
             return attach
 
-    raise ValueError("Unknown file type for %s" % filename)
+    raise ValueError('Unknown file type for %s' % filename)
 
 
 ##############################################################################

src/igvfd/memlimit.py

@@ -1,6 +1,11 @@
 # https://code.google.com/p/modwsgi/wiki/RegisteringCleanupCode
 
 
+import humanfriendly
+import psutil
+import logging
+
+
 class Generator2:
     def __init__(self, iterable, callback, environ):
         self.__iterable = iterable
@@ -33,19 +38,14 @@ def __call__(self, environ, start_response):
         return Generator2(result, self.__callback, environ)
 
 
-import logging
-import psutil
-import humanfriendly
-
-
 def rss_checker(rss_limit=None):
     log = logging.getLogger(__name__)
     process = psutil.Process()
 
     def callback(environ):
         rss = process.memory_info().rss
         if rss_limit and rss > rss_limit:
-            msg = "Restarting process. Memory usage exceeds limit of %d: %d"
+            msg = 'Restarting process. Memory usage exceeds limit of %d: %d'
             log.error(msg, rss_limit, rss)
             process.terminate()