Add CI/CD in Github Actions (#451)

Rodrigo Almeida · SRM · web-flow · commit 0dc498b81ed6 · 2022-02-08T12:29:53.000+01:00
* Add linting with flake8, black & isort - Remove python-env from tox.ini files, use gh-actions matrix for running different python versions - Define tox runs as `tox -e [lint/format/flake8/test]` * Change basepython to python3 from python3.9. Add test run with tox. * Add python3.10 to the github runner matrix * Use ubuntu-small osgeo container - Tested github-actions locally using act. - osgeo ubuntu-large images are 1.4 GB in size, takes some time to run this locally - Replacing this with ubuntu-small runs it faster with no side effects * Add release Actions workflow (#449) * Uncomment tox * Update job name to test * Add release workflow only on main and with tag * Fix flake8 (#450) * lint is flake8 * Remove - in order to not ignore exit code * Add extended ignores for flake8 * Rm unused vars * Remove extra # * Rm unused var * Add rio-cogeo dependency * Fix undefined variable, ignore ambigous name * Rm flake8 * Fix unused imports F401 * Remove - * Add cv2 dependencies * Format * Check if formatting passes * Move black/isort/flake8 config to setup.cfg * Add pre-commit config with isort, balck and flake8 * Add pre-commit dep Co-authored-by: SRM <soumya@developmentseed.org>
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,31 @@
+name: Release Python package
+
+on:
+  workflow_run:
+    workflows: [tests]
+    branches: [main]
+    types: [completed]
+
+jobs:
+  release:
+    needs: test
+    runs-on: ubuntu-latest
+    name: Release package to PyPi
+    if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
+    steps:
+      - uses: actions/checkout@master
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install tox tox-gh-actions
+      - name: Build and release 📦 to PyPI
+        with:
+          TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+        run: |
+          tox -e release
+
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -4,14 +4,14 @@ on:
   - push
 
 jobs:
-  build:
+  test:
     runs-on: ubuntu-latest
     name: Docker | GDAL=${{ matrix.gdal-version }} | python=${{ matrix.python-version }}
-    container: osgeo/gdal:ubuntu-full-${{ matrix.gdal-version }}
+    container: osgeo/gdal:ubuntu-small-${{ matrix.gdal-version }}
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9"]
+        python-version: ["3.8", "3.9", "3.10"]
         gdal-version: ["3.4.0", "3.3.3"]
         include:
           - python-version: "3.8"
@@ -38,10 +38,16 @@ jobs:
             python${{ matrix.python-version }}-dev \
             python${{ matrix.python-version }}-venv \
             python3-pip \
-            g++
+            g++ \
+            libgl1
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install tox tox-gh-actions
+          python -m pip install tox
+      - name: Lint with tox
+        run: tox -e lint
+      - name: Format with tox
+        run: tox -e format
       - name: Test with tox
-        run: tox
+        run: tox -e test
+
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,14 @@
+repos:
+-   repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+    - id: isort
+-   repo: https://github.com/ambv/black
+    rev: 22.1.0
+    hooks:
+    - id: black
+      language_version: python3.9
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
+    - id: flake8
diff --git a/docs/conf.py b/docs/conf.py
@@ -14,7 +14,7 @@
 import sys
 
 sys.path.insert(0, os.path.abspath(".."))
-import numpy as np
+import numpy as np  # noqa: F401
 import sphinx_bootstrap_theme
 
 # -- Project information -----------------------------------------------------
@@ -25,7 +25,7 @@
 license = "MIT"
 import time
 
-copyright = u"2018-{}, CosmiQ Works: an IQT Lab".format(time.strftime("%Y"))
+copyright = "2018-{}, CosmiQ Works: an IQT Lab".format(time.strftime("%Y"))
 
 # The full version, including alpha/beta/rc tags
 release = "0.0.1"
diff --git a/requirements-test.txt b/requirements-test.txt
@@ -22,3 +22,4 @@ pytest
 tox
 isort
 flake8
+pre-commit
diff --git a/setup.cfg b/setup.cfg
@@ -0,0 +1,27 @@
+[flake8]
+extend-ignore = 
+    E203, # whitespace before ':' 
+    E501, # line too long
+    C901, # is too complex
+    E402, # module level import not at top of file (for docs)
+    E741, # ambiguous variable name
+
+exclude =
+    .git,
+    __pycache__,
+    docs/source/conf.py,
+    old,
+    build,
+    dist,
+    .tox
+max-line-length = 88
+max-complexity = 10
+count=true
+
+[isort]
+profile = black
+
+[black]
+line-length = 88
+target-version = ['py37', 'py38', 'py39', 'py310']
+experimental_string_processing = true
diff --git a/setup.py b/setup.py
@@ -52,6 +52,7 @@ def check_output(cmd):
         "pyproj>=2.1",
         "PyYAML>=5.4",
         "rasterio>=1.0.23",
+        "rio-cogeo>=3.0.2",
         "requests==2.22.0",
         "rtree>=0.9.3",
         "scikit-image>=0.16.2",
@@ -79,7 +80,7 @@ def check_output(cmd):
         "Programming Language :: Python :: 3.10",
         "Topic :: Scientific/Engineering :: GIS",
     ],
-    author=u"Ryan Avery",
+    author="Ryan Avery",
     author_email="ryan@developmentseed.org",
     url="https://github.com/CosmiQ/solaris",
     license="MIT",
diff --git a/solaris/__init__.py b/solaris/__init__.py
@@ -1,4 +1,4 @@
-from . import data, utils, vector
+from . import data, utils, vector  # noqa: F401
 
 #  data, eval, preproc, raster, tile, have gdal in them need to replace with rasterio
 __version__ = "0.0.1"
diff --git a/solaris/eval/__init__.py b/solaris/eval/__init__.py
@@ -1 +1 @@
-from . import base, iou, pixel, vector
+from . import base, iou, pixel, vector  # noqa: F401
diff --git a/solaris/eval/base.py b/solaris/eval/base.py
@@ -111,8 +111,6 @@ def eval_iou_spacenet_csv(
         scoring_dict_list = []
         self.ground_truth_GDF[iou_field] = 0.0
         iou_index = self.ground_truth_GDF.columns.get_loc(iou_field)
-        id_cols = 2
-        ground_truth_ids = self.ground_truth_GDF.iloc[:, :id_cols]
 
         for imageID in tqdm(imageIDList):
             self.ground_truth_GDF_Edit = self.ground_truth_GDF[
diff --git a/solaris/preproc/__init__.py b/solaris/preproc/__init__.py
@@ -1 +1 @@
-from . import image, label, optical, pipesegment, sar
+from . import image, label, optical, pipesegment, sar  # noqa: F401
diff --git a/solaris/preproc/image.py b/solaris/preproc/image.py
@@ -9,7 +9,7 @@
 import pandas as pd
 from osgeo import gdal_array
 
-from .pipesegment import LoadSegment, MergeSegment, PipeSegment
+from .pipesegment import LoadSegment, PipeSegment
 
 
 class Image:
diff --git a/solaris/preproc/label.py b/solaris/preproc/label.py
@@ -4,7 +4,7 @@
 import shapely.wkt
 
 from ..vector.polygon import convert_poly_coords
-from .pipesegment import LoadSegment, MergeSegment, PipeSegment
+from .pipesegment import LoadSegment, PipeSegment
 
 
 class LoadString(LoadSegment):
diff --git a/solaris/preproc/optical.py b/solaris/preproc/optical.py
@@ -2,9 +2,8 @@
 
 import numpy as np
 
-from . import image
 from .image import Image
-from .pipesegment import LoadSegment, MergeSegment, PipeSegment
+from .pipesegment import PipeSegment
 
 
 class RGBToHSL(PipeSegment):
diff --git a/solaris/preproc/pipesegment.py b/solaris/preproc/pipesegment.py
@@ -412,7 +412,7 @@ def PipeFunction(inner_class=PipeSegment, pin=(), *args, **kwargs):
     and *args and **kwargs are sent to the PipeSegment's constructor.
     """
     psobject = inner_class(*args, **kwargs)
-    if issubclass(self.inner_class, LoadSegment):
+    if issubclass(self.inner_class, LoadSegment):  # noqa: F821
         return psobject()
     else:
         return (pin * psobject)()
diff --git a/solaris/preproc/sar.py b/solaris/preproc/sar.py
@@ -1,6 +1,5 @@
 import json
 import math
-import os
 import uuid
 import warnings
 import xml.etree.ElementTree as ET
@@ -12,7 +11,7 @@
 
 from . import image
 from .image import Image
-from .pipesegment import LoadSegment, MergeSegment, PipeSegment
+from .pipesegment import PipeSegment
 
 
 class BandMath(PipeSegment):
@@ -668,8 +667,8 @@ def fineoffset(self, latgrid, longrid, lattarget, lontarget, uidx, vidx):
         ulon = longrid[uidx + 1, vidx] - longrid[uidx, vidx]
         vlat = latgrid[uidx, vidx + 1] - latgrid[uidx, vidx]
         vlon = longrid[uidx, vidx + 1] - longrid[uidx, vidx]
-        uoffset = (mlat * ulat + mlon * ulon) / (ulat ** 2 + ulon ** 2)
-        voffset = (mlat * vlat + mlon * vlon) / (vlat ** 2 + vlon ** 2)
+        uoffset = (mlat * ulat + mlon * ulon) / (ulat**2 + ulon**2)
+        voffset = (mlat * vlat + mlon * vlon) / (vlat**2 + vlon**2)
         return uoffset, voffset
 
 
diff --git a/solaris/raster/__init__.py b/solaris/raster/__init__.py
@@ -1 +1 @@
-from . import image
+from . import image  # noqa: F401
diff --git a/solaris/raster/image.py b/solaris/raster/image.py
@@ -1,5 +1,3 @@
-import os
-
 import numpy as np
 import rasterio
 
diff --git a/solaris/tile/__init__.py b/solaris/tile/__init__.py
@@ -1 +1 @@
-from . import raster_tile, vector_tile
+from . import raster_tile, vector_tile  # noqa: F401
diff --git a/solaris/tile/raster_tile.py b/solaris/tile/raster_tile.py
@@ -5,10 +5,10 @@
 from rasterio.mask import mask as rasterio_mask
 from rasterio.vrt import WarpedVRT
 from rasterio.warp import Resampling, calculate_default_transform
+from rio_cogeo.cogeo import cog_translate  # ,cog_validate
 from shapely.geometry import box
 from tqdm.auto import tqdm
 
-# from rio_cogeo.cogeo import cog_validate, cog_translate
 from ..utils.core import _check_crs, _check_rasterio_im_load
 
 # removing the following until COG functionality is implemented
diff --git a/solaris/utils/__init__.py b/solaris/utils/__init__.py
@@ -1 +1 @@
-from . import cli, core, data, geo, io, tile
+from . import cli, core, data, geo, io, tile  # noqa: F401
diff --git a/solaris/utils/tile.py b/solaris/utils/tile.py
@@ -1,11 +1,5 @@
 import json
 
-import geopandas as gpd
-from affine import Affine
-from rasterio.enums import Resampling
-from rasterio.vrt import WarpedVRT
-from rasterio.windows import Window
-
 from .core import _check_crs
 
 # temporarily removing the below until I can get COG functionality implemented
diff --git a/solaris/vector/__init__.py b/solaris/vector/__init__.py
@@ -1 +1 @@
-from . import graph, mask, polygon
+from . import graph, mask, polygon  # noqa: F401
diff --git a/tests/test_cli/test_cli.py b/tests/test_cli/test_cli.py
@@ -1,8 +1,6 @@
 import os
-import pickle
 import subprocess
 
-import networkx as nx
 import numpy as np
 import skimage.io
 
diff --git a/tests/test_data/test_coco.py b/tests/test_data/test_coco.py
@@ -23,7 +23,7 @@ def test_multiclass_single_geojson(self):
             expected_dict = json.load(f)
         with open(os.path.join(data_dir, "tmp_coco.json"), "r") as f:
             saved_result = json.load(f)
-        ## Simplified test due to rounding errors- JSS
+        # Simplified test due to rounding errors- JSS
         assert (
             coco_dict["annotations"][0]["bbox"]
             == expected_dict["annotations"][0]["bbox"]
@@ -62,7 +62,7 @@ def test_singleclass_multi_geojson(self):
 
         with open(os.path.join(data_dir, "coco_sample_1.json"), "r") as f:
             expected_dict = json.load(f)
-        ## Simplified test due to rounding errors- JSS
+        # Simplified test due to rounding errors- JSS
         assert (
             expected_dict["annotations"][0]["bbox"]
             == coco_dict["annotations"][0]["bbox"]
diff --git a/tests/test_eval/evaluator_test.py b/tests/test_eval/evaluator_test.py
@@ -86,7 +86,6 @@ def test_iou_by_building(self):
         path_truth = os.path.join(data_folder, "SN2_sample_truth.csv")
         path_pred = os.path.join(data_folder, "SN2_sample_preds.csv")
         path_ious = os.path.join(data_folder, "SN2_sample_iou_by_building.csv")
-        path_temp = "./temp.pd"
         eb = Evaluator(path_truth)
         eb.load_proposal(path_pred, conf_field_list=["Confidence"], proposalCSV=True)
         eb.eval_iou_spacenet_csv(miniou=0.5, imageIDField="ImageId", min_area=20)
diff --git a/tests/test_imports.py b/tests/test_imports.py
@@ -1,3 +1,4 @@
+# flake8: noqa: F401
 class TestImports(object):
     def test_imports(self):
         import solaris
diff --git a/tests/test_raster/test_image.py b/tests/test_raster/test_image.py
@@ -1,12 +1,9 @@
 import os
 
-import numpy as np
-import skimage.io
 from affine import Affine
 
-import solaris as sol
 from solaris.data import sample_load_rasterio
-from solaris.raster.image import get_geo_transform, stitch_images
+from solaris.raster.image import get_geo_transform
 
 data_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/"))
 
diff --git a/tox.ini b/tox.ini
@@ -1,44 +1,31 @@
 [tox]
-envlist = py37,py38,py39,py310
+envlist = 
+    lint, 
+    format, 
+    test
 
-[gh-actions]
-python =
-    3.7: py37
-    3.8: py38
-    3.9: py39
-    3.10: py310
-
-[testenv]
+[testenv:test]
 extras = test
-commands=
+commands =
     python -m pytest --cov --cov-report xml --cov-report term-missing --ignore=venv
 
-# Lint
-[flake8]
-exclude = .git,__pycache__,docs/source/conf.py,old,build,dist
-max-line-length = 90
-
-[mypy]
-no_strict_optional = True
-ignore_missing_imports = True
-
-[tool:isort]
-include_trailing_comma = True
-multi_line_output = 3
-line_length = 90
-known_first_party = solaris
-default_section = THIRDPARTY
+[testenv:lint]
+basepython = python3
+skip_install = true
+deps = flake8
+commands = 
+    flake8 .
 
-# Autoformatter
-[testenv:black]
+[testenv:format]
 basepython = python3
 skip_install = true
-deps =
+deps = 
+    isort
     black
 commands =
-    black
+    isort . -c
+    black . --check
 
-# Release tooling
 [testenv:build]
 basepython = python3
 skip_install = true

-Original file line number
+Diff line change
 tox
 isort
 flake8
 +pre-commit
Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-from . import base, iou, pixel, vector`
	`1`	`+from . import base, iou, pixel, vector # noqa: F401`