Upgrade dependencies and clean up code/packaging #435
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CICD | |
| on: | |
| # Run CICD for non-draft pull request | |
| pull_request: | |
| branches: | |
| - main | |
| # Also run when the pull request merges (which generates a push) | |
| # So that we can tag the docker image appropriately. | |
| push: | |
| branches: | |
| - main | |
| - staging-* | |
| env: | |
| nexus_server: docker-registry.ign.fr | |
| jobs: | |
| build_test_deploy: | |
| runs-on: self-hosted | |
| steps: | |
| - name: Checkout branch | |
| uses: actions/checkout@v4 | |
| - name: replace BD_UNI credentials | |
| run: | | |
| cp configs/bd_uni_connection_params/credentials_template.yaml configs/bd_uni_connection_params/credentials.yaml | |
| sed -i '/user:/c\user: invite' configs/bd_uni_connection_params/credentials.yaml | |
| sed -i '/pwd:/c\pwd: ${{ secrets.PASSWORD_BD_UNI }}' configs/bd_uni_connection_params/credentials.yaml | |
| - name: build docker image | |
| run: docker build --build-arg http_proxy=http://proxy.ign.fr:3128/ --build-arg https_proxy=http://proxy.ign.fr:3128/ -t lidar_prod . | |
| - name: Check code neatness (linter) | |
| run: docker run lidar_prod flake8 | |
| - name: Run tests & get coverage - fast ones go first. | |
| run: > | |
| docker run --network host | |
| lidar_prod | |
| python -m | |
| pytest -rA -vv -m "not slow" --ignore=actions-runner | |
| - name: Run slow tests last (evaluation on large file) | |
| run: > | |
| docker run --network host | |
| -v /var/data/cicd/CICD_github_assets/M11.1/inputs/evaluation/:/lidar/tests/files/large/ | |
| lidar_prod | |
| python -m | |
| pytest -rA -v -m "slow" --ignore=actions-runner --no-cov | |
| - name: Test building module from CLI on a LAS subset. | |
| run: > | |
| docker run --network host | |
| -v /var/data/cicd/CICD_github_assets/M11.1/inputs/:/inputs/ | |
| -v /var/data/cicd/CICD_github_assets/M11.1/outputs/:/outputs/ | |
| lidar_prod | |
| python | |
| lidar_prod/run.py | |
| +task=apply_on_building | |
| paths.src_las=/inputs/Semis_2021_0937_6537_LA93_IGN69.150mx100m.for_full_building_module.las | |
| paths.output_dir=/outputs/ | |
| - name: Test vegetation/unclassified detection from CLI on a LAS subset. | |
| run: > | |
| docker run | |
| -v /var/data/cicd/CICD_github_assets/M11.1/inputs/:/inputs/ | |
| -v /var/data/cicd/CICD_github_assets/M11.1/outputs/:/outputs/ | |
| lidar_prod | |
| python | |
| lidar_prod/run.py | |
| +task=identify_vegetation_unclassified | |
| data_format=vegetation_unclassified.yaml | |
| paths.src_las=/inputs/888000_6614000.subset.las | |
| paths.output_dir=/outputs/ | |
| - name: Tag the docker image with branch name | |
| if: github.event_name == 'push' | |
| run: | | |
| docker tag lidar_prod:latest lidar_prod:${{github.ref_name}} | |
| docker run lidar_prod:${{github.ref_name}} bash # Dry run image so that is it not prunned | |
| # docker save lidar_prod:${{github.ref_name}} -o /var/data/cicd/CICD_github_assets/CICD_docker_images/lidar_prod_${{github.ref_name}}.tar # This needs writing rights to the mounted path | |
| # get version number and date, to tag the image pushed to nexus | |
| - name: Get version number | |
| id: tag | |
| run: | | |
| echo "VERSION=$(python -m lidar_prod.version)" >> $GITHUB_ENV | |
| echo "DATE=$(date '+%Y.%m.%d')" >> $GITHUB_ENV | |
| # show possible tags, for debugging purpose | |
| - name: Print tag | |
| run: | | |
| echo "${{env.VERSION}}" | |
| echo "${{env.DATE}}" | |
| - name: push main docker on nexus (tagged with a date) | |
| # we push on nexus an image from the main branch when it has been updated (push or accepted pull request) | |
| if: ((github.ref_name == 'main') && (github.event_name == 'push')) | |
| run: | | |
| docker tag lidar_prod $nexus_server/lidar_hd/lidar_prod:${{ env.VERSION }}-${{ env.DATE }} | |
| docker login $nexus_server --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }} | |
| docker push $nexus_server/lidar_hd/lidar_prod:${{ env.VERSION }}-${{ env.DATE }} | |
| - name: push branch docker on nexus (tagged with the branch name) | |
| # we push on nexus an image from a branch when it's pushed | |
| if: ((github.event_name == 'push') && (github.ref_name != 'main')) | |
| run: | | |
| docker tag lidar_prod $nexus_server/lidar_hd/lidar_prod:${{ env.VERSION }}-${{github.ref_name}} | |
| docker login $nexus_server --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }} | |
| docker push $nexus_server/lidar_hd/lidar_prod:${{ env.VERSION }}-${{github.ref_name}} | |
| - name: Clean dangling docker images | |
| if: always() # always do it, even if something failed | |
| run: docker system prune --force # remove dangling docker images, without asking user for confirmation |