Merge pull request #2100 from HubSpot/enforce_signature

Additional deploy validation

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/SingularityExecutor.java

@@ -4,6 +4,7 @@
 import com.hubspot.singularity.executor.SingularityExecutorMonitor.KillState;
 import com.hubspot.singularity.executor.SingularityExecutorMonitor.SubmitState;
 import com.hubspot.singularity.executor.config.SingularityExecutorTaskBuilder;
+import com.hubspot.singularity.executor.task.ArtifactVerificationException;
 import com.hubspot.singularity.executor.task.SingularityExecutorTask;
 import com.hubspot.singularity.executor.utils.ExecutorUtils;
 import com.hubspot.singularity.executor.utils.MesosUtils;
@@ -128,11 +129,13 @@ public void launchTask(
       }
     } catch (Throwable t) {
       LOG.error("Unexpected exception starting task {}", taskId, t);
-
+      TaskState state = t instanceof ArtifactVerificationException
+        ? TaskState.TASK_FAILED
+        : TaskState.TASK_LOST;
       executorUtils.sendStatusUpdate(
         executorDriver,
         taskInfo.getTaskId(),
-        TaskState.TASK_LOST,
+        state,
         String.format(
           "Unexpected exception while launching task %s - %s",
           taskId,

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/SingularityExecutorMonitor.java

@@ -16,6 +16,7 @@
 import com.hubspot.singularity.executor.config.SingularityExecutorConfiguration;
 import com.hubspot.singularity.executor.config.SingularityExecutorLogging;
 import com.hubspot.singularity.executor.config.SingularityExecutorModule;
+import com.hubspot.singularity.executor.task.ArtifactVerificationException;
 import com.hubspot.singularity.executor.task.SingularityExecutorTask;
 import com.hubspot.singularity.executor.task.SingularityExecutorTaskProcessCallable;
 import com.hubspot.singularity.executor.utils.ExecutorUtils;
@@ -464,11 +465,15 @@ public void onSuccess(ProcessBuilder processBuilder) {
           try {
             onSuccessThrows(processBuilder);
           } catch (Throwable t) {
+            TaskState state = t instanceof ArtifactVerificationException
+              ? TaskState.TASK_FAILED
+              : TaskState.TASK_LOST;
             finishTask(
               task,
-              TaskState.TASK_LOST,
+              state,
               String.format(
-                "Task lost while transitioning due to: %s",
+                "%s while transitioning due to: %s",
+                state,
                 t.getClass().getSimpleName()
               ),
               Optional.of("While submitting process task"),

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java

@@ -261,6 +261,8 @@ public class SingularityExecutorConfiguration extends BaseRunnerConfiguration {
   @JsonProperty
   private boolean verifyAssignedPorts = false;
 
+  private boolean failOnSignatureWithNoMatchingArtifact = false;
+
   public SingularityExecutorConfiguration() {
     super(Optional.of("singularity-executor.log"));
   }
@@ -812,6 +814,20 @@ public void setDefaultHealthcheckInternvalSeconds(
     this.defaultHealthcheckInternvalSeconds = defaultHealthcheckInternvalSeconds;
   }
 
+  public void setLogrotateHourlyConfDirectory(String logrotateHourlyConfDirectory) {
+    this.logrotateHourlyConfDirectory = logrotateHourlyConfDirectory;
+  }
+
+  public boolean isFailOnSignatureWithNoMatchingArtifact() {
+    return failOnSignatureWithNoMatchingArtifact;
+  }
+
+  public void setFailOnSignatureWithNoMatchingArtifact(
+    boolean failOnSignatureWithNoMatchingArtifact
+  ) {
+    this.failOnSignatureWithNoMatchingArtifact = failOnSignatureWithNoMatchingArtifact;
+  }
+
   @Override
   public String toString() {
     return (

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorTaskBuilder.java

@@ -37,10 +37,9 @@ public class SingularityExecutorTaskBuilder {
 
   private final SingularityRunnerBaseConfiguration baseConfiguration;
   private final SingularityExecutorConfiguration executorConfiguration;
-  private final SingularityS3Configuration s3Configuration;
   private final SingularityExecutorArtifactFetcher artifactFetcher;
   private final DockerUtils dockerUtils;
-
+  private final SingularityS3Configuration s3Configuration;
   private final SingularityExecutorLogging executorLogging;
   private final ExecutorUtils executorUtils;
 
@@ -136,8 +135,8 @@ public SingularityExecutorTask buildTask(
       log,
       jsonObjectFileHelper,
       dockerUtils,
-      s3Configuration,
-      jsonObjectMapper
+      jsonObjectMapper,
+      s3Configuration
     );
   }
 

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/ArtifactVerificationException.java

@@ -0,0 +1,8 @@
+package com.hubspot.singularity.executor.task;
+
+public class ArtifactVerificationException extends RuntimeException {
+
+  public ArtifactVerificationException(String message) {
+    super(message);
+  }
+}

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorArtifactVerifier.java

@@ -1,5 +1,6 @@
 package com.hubspot.singularity.executor.task;
 
+import com.hubspot.deploy.S3Artifact;
 import com.hubspot.deploy.S3ArtifactSignature;
 import com.hubspot.singularity.executor.config.SingularityExecutorConfiguration;
 import com.hubspot.singularity.s3.base.config.SingularityS3Configuration;
@@ -9,13 +10,14 @@
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 import org.slf4j.Logger;
 
 public class SingularityExecutorArtifactVerifier {
   private final Logger log;
   private final SingularityExecutorConfiguration executorConfiguration;
-  private final SingularityS3Configuration s3Configuration;
   private final SingularityExecutorTaskDefinition taskDefinition;
+  private final SingularityS3Configuration s3Configuration;
 
   public SingularityExecutorArtifactVerifier(
     SingularityExecutorTaskDefinition taskDefinition,
@@ -25,11 +27,14 @@ public SingularityExecutorArtifactVerifier(
   ) {
     this.log = log;
     this.executorConfiguration = executorConfiguration;
-    this.s3Configuration = s3Configuration;
     this.taskDefinition = taskDefinition;
+    this.s3Configuration = s3Configuration;
   }
 
-  public void checkSignatures(List<S3ArtifactSignature> s3ArtifactsWithSignatures) {
+  public void checkSignatures(
+    List<S3Artifact> s3Artifacts,
+    List<S3ArtifactSignature> s3ArtifactsWithSignatures
+  ) {
     if (s3ArtifactsWithSignatures.isEmpty()) {
       log.info(
         "No files containing artifact signatures specified, skipping verification."
@@ -38,18 +43,37 @@ public void checkSignatures(List<S3ArtifactSignature> s3ArtifactsWithSignatures)
     }
 
     for (S3ArtifactSignature s3ArtifactSignature : s3ArtifactsWithSignatures) {
-      checkArtifactSignature(s3ArtifactSignature);
+      Optional<S3Artifact> maybeMatchingForSignature = s3Artifacts
+        .stream()
+        .filter(s -> s3ArtifactSignature.getArtifactFilename().equals(s.getFilename()))
+        .findFirst();
+      if (maybeMatchingForSignature.isPresent()) {
+        checkArtifactSignature(maybeMatchingForSignature.get(), s3ArtifactSignature);
+      } else {
+        log.warn("No matching artifact found for signature {}", s3ArtifactSignature);
+        if (executorConfiguration.isFailOnSignatureWithNoMatchingArtifact()) {
+          throw new ArtifactVerificationException(
+            String.format(
+              "No matching artifact found for signature %s",
+              s3ArtifactSignature.getFilename()
+            )
+          );
+        }
+      }
     }
   }
 
-  private void checkArtifactSignature(S3ArtifactSignature s3ArtifactSignature) {
+  private void checkArtifactSignature(
+    S3Artifact s3Artifact,
+    S3ArtifactSignature s3ArtifactSignature
+  ) {
     final Path artifactPath = Paths.get(
       s3Configuration.getArtifactCacheDirectory(),
-      s3ArtifactSignature.getArtifactFilename()
+      s3Artifact.getFilenameForCache()
     );
     final Path artifactSignaturePath = Paths.get(
       s3Configuration.getArtifactCacheDirectory(),
-      s3ArtifactSignature.getFilename()
+      s3ArtifactSignature.getFilenameForCache()
     );
 
     if (!Files.exists(artifactPath)) {
@@ -93,7 +117,7 @@ private void checkArtifactSignature(S3ArtifactSignature s3ArtifactSignature) {
         );
 
         if (executorConfiguration.isFailTaskOnInvalidArtifactSignature()) {
-          throw new RuntimeException(
+          throw new ArtifactVerificationException(
             String.format("Failed to validate signature for artifact %s", artifactPath)
           );
         }

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTask.java

@@ -54,8 +54,8 @@ public SingularityExecutorTask(
     Logger log,
     JsonObjectFileHelper jsonObjectFileHelper,
     DockerUtils dockerUtils,
-    SingularityS3Configuration s3Configuration,
-    ObjectMapper objectMapper
+    ObjectMapper objectMapper,
+    SingularityS3Configuration s3Configuration
   ) {
     this.driver = driver;
     this.taskInfo = taskInfo;

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java

@@ -115,7 +115,10 @@ public ProcessBuilder call() throws Exception {
       );
     task
       .getArtifactVerifier()
-      .checkSignatures(executorData.getS3ArtifactSignaturesOrEmpty());
+      .checkSignatures(
+        executorData.getS3Artifacts(),
+        executorData.getS3ArtifactSignaturesOrEmpty()
+      );
 
     List<ArtifactList> artifactLists = new ArrayList<>();
     artifactLists.addAll(checkArtifactsForArtifactLists(executorData.getS3Artifacts()));
@@ -157,7 +160,7 @@ public ProcessBuilder call() throws Exception {
           s3ArtifactSignatures,
           externalArtifacts
         );
-      task.getArtifactVerifier().checkSignatures(s3ArtifactSignatures);
+      task.getArtifactVerifier().checkSignatures(s3Artifacts, s3ArtifactSignatures);
     }
 
     ProcessBuilder processBuilder = buildProcessBuilder(

SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java

@@ -422,6 +422,12 @@ public class SingularityConfiguration extends Configuration {
   @JsonProperty("crashLoop")
   private CrashLoopConfiguration crashLoopConfiguration = new CrashLoopConfiguration();
 
+  // If true, only allow artifacts with an attached signature. This disallows embedded + external artifacts, only allowing lists + s3
+  private boolean enforceSignedArtifacts = false;
+
+  // If not empty, disallow docker images in deploys unless they are in this list
+  private Set<String> validDockerRegistries = Collections.emptySet();
+
   public long getAskDriverToKillTasksAgainAfterMillis() {
     return askDriverToKillTasksAgainAfterMillis;
   }
@@ -1863,4 +1869,20 @@ public long getReconcileLaunchAfterMillis() {
   public void setReconcileLaunchAfterMillis(long reconcileLaunchAfterMillis) {
     this.reconcileLaunchAfterMillis = reconcileLaunchAfterMillis;
   }
+
+  public boolean isEnforceSignedArtifacts() {
+    return enforceSignedArtifacts;
+  }
+
+  public void setEnforceSignedArtifacts(boolean enforceSignedArtifacts) {
+    this.enforceSignedArtifacts = enforceSignedArtifacts;
+  }
+
+  public Set<String> getValidDockerRegistries() {
+    return validDockerRegistries;
+  }
+
+  public void setValidDockerRegistries(Set<String> validDockerRegistries) {
+    this.validDockerRegistries = validDockerRegistries;
+  }
 }