From ad5f0c024029fd54b0e6b85e6cae21452b12b225 Mon Sep 17 00:00:00 2001 From: benjamin-j-powell Date: Fri, 2 Feb 2024 12:33:44 -0600 Subject: [PATCH] feat(artifact): logic to handle reference URIs in overrides This commit updates the Helm util helpers that we have to include the artifact store. This will allow us to inspect the raw strings and check to see if we need to expand them. Further, I opted to not utilize the deserializer this time, since this problem seemed very specific to Helm. If that changes, we can always just use the deserializer Signed-off-by: benjamin-j-powell --- .../manifests/HelmBakeTemplateUtils.java | 56 ++++++++++++++++--- .../manifests/helm/HelmTemplateUtils.java | 14 +++-- .../helmfile/HelmfileTemplateUtils.java | 12 ++-- 3 files changed, 63 insertions(+), 19 deletions(-) diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/HelmBakeTemplateUtils.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/HelmBakeTemplateUtils.java index 5e1f99075..17f6ae070 100644 --- a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/HelmBakeTemplateUtils.java +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/HelmBakeTemplateUtils.java @@ -16,32 +16,42 @@ package com.netflix.spinnaker.rosco.manifests; -import com.netflix.spinnaker.kork.artifacts.model.Artifact; -import com.netflix.spinnaker.kork.exceptions.SpinnakerException; -import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException; import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.regex.Pattern; import java.util.stream.Collectors; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactReferenceURI; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreConfigurationProperties; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.kork.exceptions.SpinnakerException; +import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException; + +import lombok.Getter; + public abstract class HelmBakeTemplateUtils { private static final String MANIFEST_SEPARATOR = "---\n"; private static final Pattern REGEX_TESTS_MANIFESTS = Pattern.compile("# Source: .*/templates/tests/.*"); - private final ArtifactDownloader artifactDownloader; + @Getter private final ArtifactDownloader artifactDownloader; + private final ArtifactStore artifactStore; + private final ArtifactStoreConfigurationProperties.HelmConfig helmConfig; - protected HelmBakeTemplateUtils(ArtifactDownloader artifactDownloader) { + protected HelmBakeTemplateUtils( + ArtifactDownloader artifactDownloader, + Optional artifactStore, + ArtifactStoreConfigurationProperties.HelmConfig helmConfig) { this.artifactDownloader = artifactDownloader; - } - - public ArtifactDownloader getArtifactDownloader() { - return artifactDownloader; + this.artifactStore = artifactStore.orElse(null); + this.helmConfig = helmConfig; } public abstract String fetchFailureMessage(String description, Exception e); @@ -103,4 +113,32 @@ protected Path getHelmTypePathFromArtifact( return helmTypeFilePath; } + + /** + * This is a helper method to build the appropriate overrides in the event that an + * ArtifactReferenceURI was passed in an override + */ + protected List buildOverrideList(Map overrides) { + return overrides.entrySet().stream() + .map(entry -> entry.getKey() + "=" + expandArtifactReferenceURIs(entry.getValue()).toString()) + .collect(Collectors.toList()); + } + + /** + * In the event that we encounter and ArtifactReferenceURI, we want to pull + * down that artifact instead of using the raw URI as a value for helm. + */ + private Object expandArtifactReferenceURIs(Object value) { + if (artifactStore == null || !(helmConfig.isExpandOverrides() && value instanceof String)) { + return value; + } + + String ref = (String) value; + if (ArtifactReferenceURI.is(ref)) { + Artifact artifact = artifactStore.get(ArtifactReferenceURI.parse(ref)); + return artifact.getReference(); + } + + return ref; + } } diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtils.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtils.java index 3dfa3a1db..f557fb397 100644 --- a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtils.java +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtils.java @@ -1,5 +1,7 @@ package com.netflix.spinnaker.rosco.manifests.helm; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreConfigurationProperties; import com.netflix.spinnaker.kork.artifacts.model.Artifact; import com.netflix.spinnaker.rosco.jobs.BakeRecipe; import com.netflix.spinnaker.rosco.manifests.ArtifactDownloader; @@ -12,20 +14,25 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.properties.ConfigurationPropertiesScan; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; @Component @Slf4j +@ConfigurationPropertiesScan("com.netflix.spinnaker.kork.artifacts.artifactstore") public class HelmTemplateUtils extends HelmBakeTemplateUtils { private final RoscoHelmConfigurationProperties helmConfigurationProperties; public HelmTemplateUtils( ArtifactDownloader artifactDownloader, + Optional artifactStore, + ArtifactStoreConfigurationProperties artifactStoreProperties, RoscoHelmConfigurationProperties helmConfigurationProperties) { - super(artifactDownloader); + super(artifactDownloader, artifactStore, artifactStoreProperties.getHelm()); this.helmConfigurationProperties = helmConfigurationProperties; } @@ -103,10 +110,7 @@ public BakeRecipe buildCommand( Map overrides = request.getOverrides(); if (overrides != null && !overrides.isEmpty()) { - List overrideList = new ArrayList<>(); - for (Map.Entry entry : overrides.entrySet()) { - overrideList.add(entry.getKey() + "=" + entry.getValue().toString()); - } + List overrideList = buildOverrideList(overrides); String overrideOption = request.isRawOverrides() ? "--set" : "--set-string"; command.add(overrideOption); command.add(String.join(",", overrideList)); diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtils.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtils.java index 3abb66c95..f5590d593 100644 --- a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtils.java +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtils.java @@ -16,6 +16,8 @@ package com.netflix.spinnaker.rosco.manifests.helmfile; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreConfigurationProperties; import com.netflix.spinnaker.kork.artifacts.model.Artifact; import com.netflix.spinnaker.rosco.jobs.BakeRecipe; import com.netflix.spinnaker.rosco.manifests.ArtifactDownloader; @@ -28,6 +30,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -41,8 +44,10 @@ public class HelmfileTemplateUtils extends HelmBakeTemplateUtils artifactStore, + ArtifactStoreConfigurationProperties artifactStoreConfig, RoscoHelmfileConfigurationProperties helmfileConfigurationProperties) { - super(artifactDownloader); + super(artifactDownloader, artifactStore, artifactStoreConfig.getHelm()); this.helmfileConfigurationProperties = helmfileConfigurationProperties; } @@ -105,10 +110,7 @@ public BakeRecipe buildCommand( Map overrides = request.getOverrides(); if (!overrides.isEmpty()) { - List overrideList = new ArrayList<>(); - for (Map.Entry entry : overrides.entrySet()) { - overrideList.add(entry.getKey() + "=" + entry.getValue().toString()); - } + List overrideList = buildOverrideList(overrides); command.add("--set"); command.add(String.join(",", overrideList)); }