applying precommit recommandations

CSCfi · Mar 27, 2024 · 9fea7f1 · 9fea7f1
1 parent ec3c2a2
commit 9fea7f1
Show file tree

Hide file tree

Showing 33 changed files with 474 additions and 358 deletions.
diff --git a/.github/workflows/build-container-prep-image.yml b/.github/workflows/build-container-prep-image.yml
@@ -13,7 +13,7 @@ jobs:
  steps:
  - uses: actions/checkout@v4
  with:
- lfs: 'true'
+ lfs: "true"
 
  - name: Build image
  run: git lfs pull ; docker build . -f ./client/container_preparation/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
@@ -33,10 +33,10 @@ jobs:
 
  # This strips the "v" prefix from the tag name.
  [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
- 
+
  # This uses the Docker `latest` tag convention.
  [ "$VERSION" == "main" ] && VERSION=latest
  echo IMAGE_ID=$IMAGE_ID
  echo VERSION=$VERSION
  docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
- docker push $IMAGE_ID:$VERSION
+ docker push $IMAGE_ID:$VERSION
diff --git a/.github/workflows/build-data-prep-image.yml b/.github/workflows/build-data-prep-image.yml
@@ -13,7 +13,7 @@ jobs:
  steps:
  - uses: actions/checkout@v4
  with:
- lfs: 'true'
+ lfs: "true"
 
  - name: Build image
  run: git lfs pull ; docker build . -f ./client/data_preparation/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
@@ -33,10 +33,10 @@ jobs:
 
  # This strips the "v" prefix from the tag name.
  [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
- 
+
  # This uses the Docker `latest` tag convention.
  [ "$VERSION" == "main" ] && VERSION=latest
  echo IMAGE_ID=$IMAGE_ID
  echo VERSION=$VERSION
  docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
- docker push $IMAGE_ID:$VERSION
+ docker push $IMAGE_ID:$VERSION
diff --git a/.github/workflows/build-job-prep-image.yml b/.github/workflows/build-job-prep-image.yml
@@ -13,7 +13,7 @@ jobs:
  steps:
  - uses: actions/checkout@v4
  with:
- lfs: 'true'
+ lfs: "true"
 
  - name: Build image
  run: git lfs pull ; docker build . -f ./client/job_preparation/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
@@ -33,10 +33,10 @@ jobs:
 
  # This strips the "v" prefix from the tag name.
  [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
- 
+
  # This uses the Docker `latest` tag convention.
  [ "$VERSION" == "main" ] && VERSION=latest
  echo IMAGE_ID=$IMAGE_ID
  echo VERSION=$VERSION
  docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
- docker push $IMAGE_ID:$VERSION
+ docker push $IMAGE_ID:$VERSION
diff --git a/.github/workflows/build-server-image.yml b/.github/workflows/build-server-image.yml
@@ -13,7 +13,7 @@ jobs:
  steps:
  - uses: actions/checkout@v4
  with:
- lfs: 'true'
+ lfs: "true"
 
  - name: Build image
  run: git lfs pull ; docker build . -f ./server/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
@@ -33,10 +33,10 @@ jobs:
 
  # This strips the "v" prefix from the tag name.
  [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
- 
+
  # This uses the Docker `latest` tag convention.
  [ "$VERSION" == "main" ] && VERSION=latest
  echo IMAGE_ID=$IMAGE_ID
  echo VERSION=$VERSION
  docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
- docker push $IMAGE_ID:$VERSION
+ docker push $IMAGE_ID:$VERSION
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,35 +1,34 @@
 repos:
-# Base repo
--  repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v3.2.0
+ # Base repo
+ - repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v4.5.0
  hooks:
- -  id: trailing-whitespace
- -  id: end-of-file-fixer
- -  id: check-yaml
- -  id: check-added-large-files
+  - id: trailing-whitespace
+  - id: end-of-file-fixer
+  - id: check-yaml
+  - id: check-added-large-files
 
-# Code formatting using Black (python)
--  repo: https://github.com/psf/black
- rev: 24.2.0
+ # Code formatting using Black (python)
+ - repo: https://github.com/psf/black
+ rev: 24.3.0
  hooks:
- -  id: black
+  - id: black
 
-# Dockerfile lint
-- repo: https://github.com/pryorda/dockerfilelint-precommit-hooks
- rev: v0.1.0
- hooks:
- - id: dockerfilelint
- stages: [commit]
+ # Dockerfile lint
+ - repo: https://github.com/hadolint/hadolint
+ rev: v2.12.1-beta
+ hooks:
+ - id: hadolint
 
-# Code formatting using beautysh (bash)
-- repo: https://github.com/lovesegfault/beautysh
- rev: v6.2.1
- hooks:
- - id: beautysh
+ # Code formatting using beautysh (bash)
+ - repo: https://github.com/scop/pre-commit-shfmt
+  rev: v3.8.0-1
+  hooks:
+  - id: shfmt
 
-# Markdown lint
-- repo: https://github.com/igorshubovych/markdownlint-cli
- rev: v0.39.0
- hooks:
- - id: markdownlint
-
+ # Markdown lint
+ - repo: https://github.com/pre-commit/mirrors-prettier
+  rev: v4.0.0-alpha.8
+  hooks:
+  - id: prettier
+  files: \.(js|ts|jsx|tsx|css|less|html|json|markdown|md|yaml|yml)$
diff --git a/LICENSE b/LICENSE
@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.
diff --git a/README.md b/README.md
@@ -3,6 +3,7 @@
 ## Main goal
 
 This partnership project involving CSC and Hewlett Packard Enterprise aims to enable HPC users to run secured jobs. It provides tools to enable anyone running secured jobs with encrypted data and specific confidential containers on a supercomputing site, leveraging (non exhaustively) :
+
 - [SPIFFE/SPIRE](https://github.com/spiffe/spire)
 - [Hashicorp Vault](https://github.com/hashicorp/vault)
 - [Singularity / Apptainer encryption](https://github.com/apptainer/apptainer)

diff --git a/client/container_preparation/Dockerfile b/client/container_preparation/Dockerfile
@@ -1,39 +1,39 @@
 # Using Python original Docker image
-FROM --platform=linux/amd64 python:3.9-alpine
-
-# Install necessary packages
-RUN apk add \
- git \
- curl \
- jq \
- build-base \
- libffi-dev
-
-RUN curl https://sh.rustup.rs -sSf -o rustup.sh ; chmod +x rustup.sh ; ./rustup.sh -y
+ARG BUILDPLATFORM=linux/amd64
+FROM --platform=$BUILDPLATFORM python:3.9-alpine
+
+# Install necessary packages, spire-agent and rust
+RUN apk add --no-cache \
+ git=2.43.0-r0 \
+ curl=8.5.0-r0 \
+ jq=1.7.1-r0 \
+ build-base=0.5-r3 \
+ libffi-dev=3.4.4-r3 && \
+curl -LsSf -o spire-1.9.0-linux-amd64-musl.tar.gz https://github.com/spiffe/spire/releases/download/v1.9.0/spire-1.9.0-linux-amd64-musl.tar.gz && \
+tar xvf spire-1.9.0-linux-amd64-musl.tar.gz ; mv spire-1.9.0 /opt ; mv /opt/spire-1.9.0 /opt/spire && \
+ln -s /opt/spire/bin/spire-agent /usr/bin/spire-agent && \
+ln -s /opt/spire/bin/spire-server /usr/bin/spire-server && \
+rm -rf spire-1.9.0-linux-amd64-musl.tar.gz && \
+curl https://sh.rustup.rs -sSf -o rustup.sh ; chmod +x rustup.sh ; ./rustup.sh -y
+
+# Add rust binaries to PATH
 ENV PATH="$PATH:/root/.cargo/bin"
 
-# Install spire-agent
-RUN wget -q https://github.com/spiffe/spire/releases/download/v1.9.1/spire-1.9.1-linux-amd64-musl.tar.gz
-RUN tar xvf spire-1.9.1-linux-amd64-musl.tar.gz ; mv spire-1.9.1 /opt ; mv /opt/spire-1.9.1 /opt/spire
-RUN ln -s /opt/spire/bin/spire-agent /usr/bin/spire-agent
-
-# Install pyspiffe package
-RUN pip install git+https://github.com/HewlettPackard/py-spiffe.git@3640af9d6629c05e027f99010abc934cb74122a8
-
 # Create code directory, output directory
-RUN mkdir /container_preparation /output ; chmod -R 777 /output
+RUN mkdir /container_preparation /output
 
 # Copy useful data from the project
 COPY ./client/container_preparation /container_preparation
 
+# Set workdir
+WORKDIR /container_preparation
+
 # Install dependencies
-RUN cd /container_preparation && pip install -r ./requirements.txt
+RUN pip install --no-cache-dir -r ./requirements.txt && \
+pip install --no-cache-dir git+https://github.com/HewlettPackard/py-spiffe.git@3640af9d6629c05e027f99010abc934cb74122a8
 
 # Copy utils for SPIFFEID creation ...
 COPY ./utils /container_preparation/utils
 
-# Set workdir
-WORKDIR /container_preparation
-
 # Set entrypoint
 ENTRYPOINT [ "./entrypoint.sh" ]
diff --git a/client/container_preparation/README.md b/client/container_preparation/README.md
@@ -1,18 +1,20 @@
 # Introduction
+
 This directory contains code which prepares existing OCI images to be used on LUMI in a secure way.
 The code adds layers to handle encryption and encrypts the resulting apptainer (singularity) image itself.
 
 ## Current state
 
 Currently, the container_preparation.py script is able to run most of the needed tasks
+
 - Create a new receipe (Dockerfile) prepared for secure workloads
 - Build the new image
 - Build an apptainer image based on the just built one
- - Unencrypted
- - But unfortunately not encrypted for the moment
-
+ - Unencrypted
+ - But unfortunately not encrypted for the moment
 
 What is missing :
+
 - Encryption of the container
 - Crypt binary inside of the resulting container and the logic needed to encrypt ouput data before leaving the container
 - Documentation (global) - Explanation of how it works, what is needed ...