applying precommit recommandations

CSCfi · Mar 27, 2024 · 1b5bd11 · 1b5bd11
1 parent ec3c2a2
commit 1b5bd11
Show file tree

Hide file tree

Showing 33 changed files with 468 additions and 359 deletions.
diff --git a/.github/workflows/build-container-prep-image.yml b/.github/workflows/build-container-prep-image.yml
@@ -13,7 +13,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          lfs: 'true'
+          lfs: "true"
 
       - name: Build image
         run: git lfs pull ; docker build . -f ./client/container_preparation/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
@@ -33,10 +33,10 @@ jobs:
 
           # This strips the "v" prefix from the tag name.
           [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
-          
+
           # This uses the Docker `latest` tag convention.
           [ "$VERSION" == "main" ] && VERSION=latest
           echo IMAGE_ID=$IMAGE_ID
           echo VERSION=$VERSION
           docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION
diff --git a/.github/workflows/build-data-prep-image.yml b/.github/workflows/build-data-prep-image.yml
@@ -13,7 +13,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          lfs: 'true'
+          lfs: "true"
 
       - name: Build image
         run: git lfs pull ; docker build . -f ./client/data_preparation/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
@@ -33,10 +33,10 @@ jobs:
 
           # This strips the "v" prefix from the tag name.
           [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
-          
+
           # This uses the Docker `latest` tag convention.
           [ "$VERSION" == "main" ] && VERSION=latest
           echo IMAGE_ID=$IMAGE_ID
           echo VERSION=$VERSION
           docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION
diff --git a/.github/workflows/build-job-prep-image.yml b/.github/workflows/build-job-prep-image.yml
@@ -13,7 +13,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          lfs: 'true'
+          lfs: "true"
 
       - name: Build image
         run: git lfs pull ; docker build . -f ./client/job_preparation/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
@@ -33,10 +33,10 @@ jobs:
 
           # This strips the "v" prefix from the tag name.
           [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
-          
+
           # This uses the Docker `latest` tag convention.
           [ "$VERSION" == "main" ] && VERSION=latest
           echo IMAGE_ID=$IMAGE_ID
           echo VERSION=$VERSION
           docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION
diff --git a/.github/workflows/build-server-image.yml b/.github/workflows/build-server-image.yml
@@ -13,7 +13,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          lfs: 'true'
+          lfs: "true"
 
       - name: Build image
         run: git lfs pull ; docker build . -f ./server/Dockerfile -t $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
@@ -33,10 +33,10 @@ jobs:
 
           # This strips the "v" prefix from the tag name.
           [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
-          
+
           # This uses the Docker `latest` tag convention.
           [ "$VERSION" == "main" ] && VERSION=latest
           echo IMAGE_ID=$IMAGE_ID
           echo VERSION=$VERSION
           docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,35 +1,34 @@
 repos:
-# Base repo
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
+  # Base repo
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
     hooks:
-    -   id: trailing-whitespace
-    -   id: end-of-file-fixer
-    -   id: check-yaml
-    -   id: check-added-large-files
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
 
-# Code formatting using Black (python)
--   repo: https://github.com/psf/black
-    rev: 24.2.0
+  # Code formatting using Black (python)
+  - repo: https://github.com/psf/black
+    rev: 24.3.0
     hooks:
-    -   id: black
+      - id: black
 
-# Dockerfile lint
-- repo: https://github.com/pryorda/dockerfilelint-precommit-hooks
-  rev: v0.1.0
-  hooks:
-  - id: dockerfilelint
-    stages: [commit]
+  # Dockerfile lint
+  - repo: https://github.com/hadolint/hadolint
+    rev: v2.12.1-beta
+    hooks:
+      - id: hadolint
 
-# Code formatting using beautysh (bash)
-- repo: https://github.com/lovesegfault/beautysh
-  rev: v6.2.1
-  hooks:
-    - id: beautysh
+  # Code formatting using beautysh (bash)
+  - repo: https://github.com/scop/pre-commit-shfmt
+    rev: v3.8.0-1
+    hooks:
+      - id: shfmt
 
-# Markdown lint
-- repo: https://github.com/igorshubovych/markdownlint-cli
-  rev: v0.39.0
-  hooks:
-  - id: markdownlint
-
+  # Markdown lint
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v4.0.0-alpha.8
+    hooks:
+      - id: prettier
+        files: \.(js|ts|jsx|tsx|css|less|html|json|markdown|md|yaml|yml)$
diff --git a/LICENSE b/LICENSE
@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.
diff --git a/README.md b/README.md
@@ -3,6 +3,7 @@
 ## Main goal
 
 This partnership project involving CSC and Hewlett Packard Enterprise aims to enable HPC users to run secured jobs. It provides tools to enable anyone running secured jobs with encrypted data and specific confidential containers on a supercomputing site, leveraging (non exhaustively) :
+
 - [SPIFFE/SPIRE](https://github.com/spiffe/spire)
 - [Hashicorp Vault](https://github.com/hashicorp/vault)
 - [Singularity / Apptainer encryption](https://github.com/apptainer/apptainer)

diff --git a/client/container_preparation/Dockerfile b/client/container_preparation/Dockerfile
@@ -1,39 +1,40 @@
 # Using Python original Docker image
-FROM --platform=linux/amd64 python:3.9-alpine
+ARG  BUILDPLATFORM=linux/amd64
+FROM --platform=$BUILDPLATFORM python:3.9-alpine
 
-# Install necessary packages
-RUN apk add \
-    git \
-    curl \
-    jq \
-    build-base \
-    libffi-dev
 
-RUN curl https://sh.rustup.rs -sSf -o rustup.sh ; chmod +x rustup.sh ; ./rustup.sh -y
-ENV PATH="$PATH:/root/.cargo/bin"
-
-# Install spire-agent
-RUN wget -q https://github.com/spiffe/spire/releases/download/v1.9.1/spire-1.9.1-linux-amd64-musl.tar.gz
-RUN tar xvf spire-1.9.1-linux-amd64-musl.tar.gz ; mv spire-1.9.1 /opt ; mv /opt/spire-1.9.1 /opt/spire
-RUN ln -s /opt/spire/bin/spire-agent /usr/bin/spire-agent
 
-# Install pyspiffe package
-RUN pip install git+https://github.com/HewlettPackard/py-spiffe.git@3640af9d6629c05e027f99010abc934cb74122a8
+# Add rust binaries to PATH
+ENV PATH="$PATH:/root/.cargo/bin"
 
 # Create code directory, output directory
-RUN mkdir /container_preparation /output ; chmod -R 777 /output
+RUN mkdir /container_preparation /output
 
 # Copy useful data from the project
 COPY ./client/container_preparation /container_preparation
 
-# Install dependencies
-RUN cd /container_preparation && pip install -r ./requirements.txt
-
 # Copy utils for SPIFFEID creation ...
 COPY ./utils /container_preparation/utils
 
 # Set workdir
 WORKDIR /container_preparation
 
+# Install necessary packages, spire-agent and rust
+RUN apk add --no-cache \
+    git=2.43.0-r0 \
+    curl=8.5.0-r0 \
+    jq=1.7.1-r0 \
+    build-base=0.5-r3 \
+    libffi-dev=3.4.4-r3 && \
+curl -LsSf -o spire-1.9.0-linux-amd64-musl.tar.gz https://github.com/spiffe/spire/releases/download/v1.9.0/spire-1.9.0-linux-amd64-musl.tar.gz && \
+tar xvf spire-1.9.0-linux-amd64-musl.tar.gz ; mv spire-1.9.0 /opt ; mv /opt/spire-1.9.0 /opt/spire && \
+ln -s /opt/spire/bin/spire-agent /usr/bin/spire-agent && \
+ln -s /opt/spire/bin/spire-server /usr/bin/spire-server && \
+rm -rf spire-1.9.0-linux-amd64-musl.tar.gz && \
+curl https://sh.rustup.rs -sSf -o rustup.sh ; chmod +x rustup.sh ; ./rustup.sh -y ; export PATH="$PATH":/root/.cargo/bin && \
+pip install --no-cache-dir -r ./requirements.txt && \
+pip install --no-cache-dir git+https://github.com/HewlettPackard/py-spiffe.git@3640af9d6629c05e027f99010abc934cb74122a8 && \
+rm -r /root/.cargo /root/.rustup
+
 # Set entrypoint
 ENTRYPOINT [ "./entrypoint.sh" ]
diff --git a/client/container_preparation/README.md b/client/container_preparation/README.md
@@ -1,18 +1,20 @@
 # Introduction
+
 This directory contains code which prepares existing OCI images to be used on LUMI in a secure way.
 The code adds layers to handle encryption and encrypts the resulting apptainer (singularity) image itself.
 
 ## Current state
 
 Currently, the container_preparation.py script is able to run most of the needed tasks
+
 - Create a new receipe (Dockerfile) prepared for secure workloads
 - Build the new image
 - Build an apptainer image based on the just built one
-    - Unencrypted
-    - But unfortunately not encrypted for the moment
-
+  - Unencrypted
+  - But unfortunately not encrypted for the moment
 
 What is missing :
+
 - Encryption of the container
 - Crypt binary inside of the resulting container and the logic needed to encrypt ouput data before leaving the container
 - Documentation (global) - Explanation of how it works, what is needed ...