Refactoring on python deps generation.

freenowtech · Apr 12, 2024 · 7631168 · 7631168
1 parent 8827c75
commit 7631168
Show file tree

Hide file tree

Showing 7 changed files with 70 additions and 50 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -28,11 +28,15 @@ jobs:
           python-version: "3.11.4"
       - run: python --version
 
-      - name: Set up Poetry
+      - name: Set up Poetry and CycloneDX
         uses: Gr1N/setup-poetry@v8
         with:
           poetry-version: "1.1.7"
       - run: poetry --version
+      - run: pipx --version
+      - run: pipx ensurepath
+      - run: pipx inject poetry poetry-plugin-export
+      - run: pipx install cyclonedx-bom
 
       - name: Build with Maven and run the tests
         run: mvn --batch-mode --update-snapshots verify -Dgpg.skip=true
diff --git a/plugins/dependency-checker/README.md b/plugins/dependency-checker/README.md
@@ -43,7 +43,19 @@ This plugin requires the below dependencies to be executed locally:
 ```bash
 brew install python@3.11.4 
 ``` 
-* Poetry
+* pipx - Install and Run Python Applications in Isolated Environments
 ```bash
-python -m pip install poetry==1.1.7
+brew install pipx
+pipx ensurepath
+```
+
+* Poetry and Poetry Export Plugin
+```bash
+pipx install poetry
+pipx inject poetry poetry-plugin-export
+```
+
+* CycloneDX Python SBOM Generation Tool
+```bash
+pipx install cyclonedx-bom
 ```
diff --git a/plugins/dependency-checker/src/main/java/com/freenow/sauron/plugins/command/Command.java b/plugins/dependency-checker/src/main/java/com/freenow/sauron/plugins/command/Command.java
@@ -1,8 +1,10 @@
 package com.freenow.sauron.plugins.command;
 
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.nio.file.Path;
 import java.util.List;
@@ -17,7 +19,6 @@
 @Builder
 public class Command
 {
-
     public static final String BIN_BASH = "/bin/bash";
     public static final String BASH_C_OPTION = "-c";
     public static final String AND = " && ";
@@ -46,6 +47,13 @@ public void run() throws IOException, InterruptedException, NonZeroExitCodeExcep
             throw new NonZeroExitCodeException(commandline.toString(), new String(process.getErrorStream().readAllBytes()));
         }
 
+        BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
+        String line;
+        while ((line = reader.readLine()) != null)
+        {
+            log.info(line);
+        }
+
         String processLogs = new String(process.getErrorStream().readAllBytes());
         if (isNotBlank(processLogs))
         {

diff --git a/.../src/main/java/com/freenow/sauron/plugins/generator/python/PythonDependencyGenerator.java b/.../src/main/java/com/freenow/sauron/plugins/generator/python/PythonDependencyGenerator.java
@@ -1,27 +1,22 @@
 package com.freenow.sauron.plugins.generator.python;
 
-import com.freenow.sauron.plugins.command.Command;
 import com.freenow.sauron.plugins.command.NonZeroExitCodeException;
 import com.freenow.sauron.plugins.generator.DependencyGenerator;
 import com.freenow.sauron.properties.PluginsConfigurationProperties;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.List;
 import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 public abstract class PythonDependencyGenerator extends DependencyGenerator
 {
-    protected static final String REQUIREMENTS_FREEZE_FILE = "requirements.freeze";
-    protected static final String PIP_INSTALL_CYCLONE_DX_BOM = "python -m pip install --target env cyclonedx-bom";
-    protected static final String CYCLONE_DX_GENERATE_BOM = "python -m cyclonedx_py requirements ../" + REQUIREMENTS_FREEZE_FILE + " --of XML -o ../bom.xml";
-    protected static final String PYTHON_VIRTUAL_ENV_CREATE = "-m venv .";
+
+    protected static final String PYTHON_VIRTUAL_ENV_CREATE = "python -m venv .";
     protected static final String PYTHON_VIRTUAL_ENV_ACTIVATE = "source bin/activate";
+    protected static final String CYCLONE_DX_GENERATE_BOM = "cyclonedx-py requirements requirements.freeze --of XML -o bom.xml";
     protected static final String PYTHON_VIRTUAL_ENV_DEACTIVATE = "deactivate";
-    protected static final String GO_TO_ENV = "cd env/ ";
-    protected static final String RETURN = "cd .. ";
-
     protected String python = "python";
 
 
@@ -49,36 +44,26 @@ public Path generateCycloneDxBom(Path repositoryPath)
     {
         try
         {
-            createPythonVirtualEnv(repositoryPath);
             generateRequirementsFreeze(repositoryPath);
-
-            return repositoryPath.resolve("bom.xml");
+            Path resolved = repositoryPath.resolve("bom.xml");
+            if (resolved.toFile().exists() && Files.size(resolved) > 0)
+            {
+                log.info("BOM file created.");
+            }
+            else
+            {
+                log.info("BOM file is either empty or does not exist.");
+            }
+            return resolved;
         }
         catch (Exception e)
         {
             log.error("Skip building Python Cyclone DX BOM: {}", e.getMessage());
         }
-
         return null;
     }
 
 
-    private void createPythonVirtualEnv(Path repositoryPath) throws IOException, InterruptedException
-    {
-        Command.builder()
-            .commandTimeout(commandTimeoutMinutes)
-            .repositoryPath(repositoryPath)
-            .commandline(pythonCommand(PYTHON_VIRTUAL_ENV_CREATE))
-            .build()
-            .run();
-    }
-
-
     protected abstract void generateRequirementsFreeze(Path repositoryPath) throws IOException, InterruptedException, NonZeroExitCodeException;
 
-
-    protected List<String> pythonCommand(String parameters)
-    {
-        return List.of(python.concat(" ").concat(parameters).split("\\s+"));
-    }
 }
diff --git a/...ain/java/com/freenow/sauron/plugins/generator/python/PythonPoetryDependencyGenerator.java b/...ain/java/com/freenow/sauron/plugins/generator/python/PythonPoetryDependencyGenerator.java
@@ -32,12 +32,12 @@ protected void generateRequirementsFreeze(Path repositoryPath)
                 .commandTimeout(commandTimeoutMinutes)
                 .repositoryPath(repositoryPath)
                 .commandline(
-                    List.of(BIN_BASH, BASH_C_OPTION,
+                    List.of(
+                        BIN_BASH, BASH_C_OPTION,
+                        PYTHON_VIRTUAL_ENV_CREATE + AND +
                         PYTHON_VIRTUAL_ENV_ACTIVATE + AND +
                         POETRY_EXPORT + AND +
-                        PIP_INSTALL_CYCLONE_DX_BOM + AND +
-                        GO_TO_ENV + AND + CYCLONE_DX_GENERATE_BOM + AND +
-                        RETURN + AND +
+                        CYCLONE_DX_GENERATE_BOM + AND +
                         PYTHON_VIRTUAL_ENV_DEACTIVATE
                     )
                 )

diff --git a/...va/com/freenow/sauron/plugins/generator/python/PythonRequirementsDependencyGenerator.java b/...va/com/freenow/sauron/plugins/generator/python/PythonRequirementsDependencyGenerator.java
@@ -14,8 +14,7 @@
 @Slf4j
 public class PythonRequirementsDependencyGenerator extends PythonDependencyGenerator
 {
-    private static final String PIP_INSTALL_COMMAND = "python -m pip install -r requirements.txt --target env";
-    private static final String FREEZE_COMMAND = "python -m pip freeze --path env > requirements.freeze";
+    private static final String CYCLONE_DX_GENERATE_BOM = "cyclonedx-py requirements requirements.txt --of XML -o bom.xml";
 
 
     public PythonRequirementsDependencyGenerator(PluginsConfigurationProperties properties)
@@ -33,13 +32,11 @@ protected void generateRequirementsFreeze(Path repositoryPath)
                 .commandTimeout(commandTimeoutMinutes)
                 .repositoryPath(repositoryPath)
                 .commandline(
-                    List.of(BIN_BASH, BASH_C_OPTION,
+                    List.of(
+                        BIN_BASH, BASH_C_OPTION,
+                        PYTHON_VIRTUAL_ENV_CREATE + AND +
                         PYTHON_VIRTUAL_ENV_ACTIVATE + AND +
-                        PIP_INSTALL_COMMAND + AND +
-                        FREEZE_COMMAND + AND +
-                        PIP_INSTALL_CYCLONE_DX_BOM + AND +
-                        GO_TO_ENV + AND + CYCLONE_DX_GENERATE_BOM + AND +
-                        RETURN + AND +
+                        CYCLONE_DX_GENERATE_BOM + AND +
                         PYTHON_VIRTUAL_ENV_DEACTIVATE
                     )
                 )

diff --git a/sauron-service/Dockerfile b/sauron-service/Dockerfile
@@ -53,14 +53,27 @@ RUN curl -s https://pyenv.run | bash && eval "$(/root/.pyenv/bin/pyenv init -)"
 
 RUN update-alternatives --install /usr/bin/python python /root/.pyenv/versions/"$PYTHON_VERSION"/bin/python 10
 
-# Installing Poetry
-RUN python -m pip install poetry==1.1.7
-
 # Upgrade Pip
 RUN python -m pip install --upgrade pip
 
-# Enabling Poetry to be used without need of full path 
-RUN ln -s ~/.pyenv/versions/"$PYTHON_VERSION"/bin/poetry /usr/local/bin/poetry
+# Installing pipx - https://pipx.pypa.io/stable/
+RUN python -m pip install --user pipx
+RUN python -m pipx ensurepath
+
+# Installing Poetry
+RUN python -m pipx install poetry
+
+# Installing Poetry Export Plugin
+RUN python -m pipx inject poetry poetry-plugin-export
+
+# Installing CycloneDX Python SBOM Generation Tool
+RUN python -m pipx install cyclonedx-bom
+
+# Enabling Poetry to be used without need of full path
+ RUN ln -s /root/.local/bin/poetry /usr/local/bin/poetry
+
+# Enabling CycloneDX to be used without need of full path
+ RUN ln -s /root/.local/bin/cyclonedx-py /usr/local/bin/cyclonedx-py
 
 RUN mkdir /root/.m2
 
@@ -90,6 +103,7 @@ RUN chmod a+x /usr/bin/tini
 COPY docker/entrypoint.sh /usr/local/bin/entrypoint
 
 COPY target/sauron-service.jar /sauron
+COPY target/pyproject.toml /sauron
 
 EXPOSE 8080