Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(artifact): Introduces the expansion of artifact store references in parameters #1062

Closed
wants to merge 2 commits into from
Closed

feat(artifact): Introduces the expansion of artifact store references in parameters #1062

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ad5f0c0
feat(artifact): logic to handle reference URIs in overrides
Feb 2, 2024
f9e9b9a
feat(artifacts): Updates to tests due to Helm util constructor changes
Feb 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 47 additions & 9 deletions ...-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/HelmBakeTemplateUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,32 +16,42 @@

package com.netflix.spinnaker.rosco.manifests;

import com.netflix.spinnaker.kork.artifacts.model.Artifact;
import com.netflix.spinnaker.kork.exceptions.SpinnakerException;
import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactReferenceURI;
import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore;
import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreConfigurationProperties;
import com.netflix.spinnaker.kork.artifacts.model.Artifact;
import com.netflix.spinnaker.kork.exceptions.SpinnakerException;
import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException;

import lombok.Getter;

public abstract class HelmBakeTemplateUtils<T extends BakeManifestRequest> {
private static final String MANIFEST_SEPARATOR = "---\n";
private static final Pattern REGEX_TESTS_MANIFESTS =
Pattern.compile("# Source: .*/templates/tests/.*");

private final ArtifactDownloader artifactDownloader;
@Getter private final ArtifactDownloader artifactDownloader;
private final ArtifactStore artifactStore;
private final ArtifactStoreConfigurationProperties.HelmConfig helmConfig;

protected HelmBakeTemplateUtils(ArtifactDownloader artifactDownloader) {
protected HelmBakeTemplateUtils(
ArtifactDownloader artifactDownloader,
Optional<ArtifactStore> artifactStore,
ArtifactStoreConfigurationProperties.HelmConfig helmConfig) {
this.artifactDownloader = artifactDownloader;
}

public ArtifactDownloader getArtifactDownloader() {
return artifactDownloader;
this.artifactStore = artifactStore.orElse(null);
this.helmConfig = helmConfig;
}

public abstract String fetchFailureMessage(String description, Exception e);
Expand Down Expand Up @@ -103,4 +113,32 @@ protected Path getHelmTypePathFromArtifact(

return helmTypeFilePath;
}

/**
* This is a helper method to build the appropriate overrides in the event that an
* ArtifactReferenceURI was passed in an override
*/
protected List<String> buildOverrideList(Map<String, Object> overrides) {
return overrides.entrySet().stream()
.map(entry -> entry.getKey() + "=" + expandArtifactReferenceURIs(entry.getValue()).toString())
.collect(Collectors.toList());
}

/**
* In the event that we encounter and ArtifactReferenceURI, we want to pull
* down that artifact instead of using the raw URI as a value for helm.
*/
private Object expandArtifactReferenceURIs(Object value) {
if (artifactStore == null || !(helmConfig.isExpandOverrides() && value instanceof String)) {
return value;
}

String ref = (String) value;
if (ArtifactReferenceURI.is(ref)) {
Artifact artifact = artifactStore.get(ArtifactReferenceURI.parse(ref));
return artifact.getReference();
}

return ref;
}
}
14 changes: 9 additions & 5 deletions ...manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.netflix.spinnaker.rosco.manifests.helm;

import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore;
import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreConfigurationProperties;
import com.netflix.spinnaker.kork.artifacts.model.Artifact;
import com.netflix.spinnaker.rosco.jobs.BakeRecipe;
import com.netflix.spinnaker.rosco.manifests.ArtifactDownloader;
Expand All @@ -12,20 +14,25 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.properties.ConfigurationPropertiesScan;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;

@Component
@Slf4j
@ConfigurationPropertiesScan("com.netflix.spinnaker.kork.artifacts.artifactstore")
public class HelmTemplateUtils extends HelmBakeTemplateUtils<HelmBakeManifestRequest> {
private final RoscoHelmConfigurationProperties helmConfigurationProperties;

public HelmTemplateUtils(
ArtifactDownloader artifactDownloader,
Optional<ArtifactStore> artifactStore,
ArtifactStoreConfigurationProperties artifactStoreProperties,
RoscoHelmConfigurationProperties helmConfigurationProperties) {
super(artifactDownloader);
super(artifactDownloader, artifactStore, artifactStoreProperties.getHelm());
this.helmConfigurationProperties = helmConfigurationProperties;
}

Expand Down Expand Up @@ -103,10 +110,7 @@ public BakeRecipe buildCommand(

Map<String, Object> overrides = request.getOverrides();
if (overrides != null && !overrides.isEmpty()) {
List<String> overrideList = new ArrayList<>();
for (Map.Entry<String, Object> entry : overrides.entrySet()) {
overrideList.add(entry.getKey() + "=" + entry.getValue().toString());
}
List<String> overrideList = buildOverrideList(overrides);
String overrideOption = request.isRawOverrides() ? "--set" : "--set-string";
command.add(overrideOption);
command.add(String.join(",", overrideList));
Expand Down
12 changes: 7 additions & 5 deletions ...s/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@

package com.netflix.spinnaker.rosco.manifests.helmfile;

import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore;
import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreConfigurationProperties;
import com.netflix.spinnaker.kork.artifacts.model.Artifact;
import com.netflix.spinnaker.rosco.jobs.BakeRecipe;
import com.netflix.spinnaker.rosco.manifests.ArtifactDownloader;
Expand All @@ -28,6 +30,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
Expand All @@ -41,8 +44,10 @@ public class HelmfileTemplateUtils extends HelmBakeTemplateUtils<HelmfileBakeMan

public HelmfileTemplateUtils(
ArtifactDownloader artifactDownloader,
Optional<ArtifactStore> artifactStore,
ArtifactStoreConfigurationProperties artifactStoreConfig,
RoscoHelmfileConfigurationProperties helmfileConfigurationProperties) {
super(artifactDownloader);
super(artifactDownloader, artifactStore, artifactStoreConfig.getHelm());
this.helmfileConfigurationProperties = helmfileConfigurationProperties;
}

Expand Down Expand Up @@ -105,10 +110,7 @@ public BakeRecipe buildCommand(

Map<String, Object> overrides = request.getOverrides();
if (!overrides.isEmpty()) {
List<String> overrideList = new ArrayList<>();
for (Map.Entry<String, Object> entry : overrides.entrySet()) {
overrideList.add(entry.getKey() + "=" + entry.getValue().toString());
}
List<String> overrideList = buildOverrideList(overrides);
command.add("--set");
command.add(String.join(",", overrideList));
}
Expand Down
95 changes: 85 additions & 10 deletions ...fests/src/test/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore;
import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreConfigurationProperties;
import com.netflix.spinnaker.kork.artifacts.model.Artifact;
import com.netflix.spinnaker.kork.exceptions.SpinnakerException;
import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException;
Expand All @@ -47,6 +49,9 @@
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
Expand All @@ -70,14 +75,27 @@ final class HelmTemplateUtilsTest {

private HelmBakeManifestRequest bakeManifestRequest;

/**
* Configuration for the default values in the event artifact store had been
* turned off.
*/
private ArtifactStoreConfigurationProperties artifactStoreConfig;

@BeforeEach
private void init(TestInfo testInfo) {
System.out.println("--------------- Test " + testInfo.getDisplayName());

artifactDownloader = mock(ArtifactDownloader.class);
RoscoHelmConfigurationProperties helmConfigurationProperties =
new RoscoHelmConfigurationProperties();
helmTemplateUtils = new HelmTemplateUtils(artifactDownloader, helmConfigurationProperties);
artifactStoreConfig = new ArtifactStoreConfigurationProperties();
ArtifactStoreConfigurationProperties.HelmConfig helmConfig =
new ArtifactStoreConfigurationProperties.HelmConfig();
artifactStoreConfig.setHelm(helmConfig);
helmConfig.setExpandOverrides(false);
helmTemplateUtils =
new HelmTemplateUtils(
artifactDownloader, Optional.empty(), artifactStoreConfig, helmConfigurationProperties);
Artifact chartArtifact = Artifact.builder().name("test-artifact").version("3").build();

bakeManifestRequest = new HelmBakeManifestRequest();
Expand Down Expand Up @@ -154,7 +172,8 @@ public void removeTestsDirectoryTemplatesWithTests() throws IOException {
RoscoHelmConfigurationProperties helmConfigurationProperties =
new RoscoHelmConfigurationProperties();
HelmTemplateUtils helmTemplateUtils =
new HelmTemplateUtils(artifactDownloader, helmConfigurationProperties);
new HelmTemplateUtils(
artifactDownloader, Optional.empty(), artifactStoreConfig, helmConfigurationProperties);

String output = helmTemplateUtils.removeTestsDirectoryTemplates(inputManifests);

Expand Down Expand Up @@ -208,7 +227,8 @@ public void removeTestsDirectoryTemplatesWithoutTests() throws IOException {
RoscoHelmConfigurationProperties helmConfigurationProperties =
new RoscoHelmConfigurationProperties();
HelmTemplateUtils helmTemplateUtils =
new HelmTemplateUtils(artifactDownloader, helmConfigurationProperties);
new HelmTemplateUtils(
artifactDownloader, Optional.empty(), artifactStoreConfig, helmConfigurationProperties);

String output = helmTemplateUtils.removeTestsDirectoryTemplates(inputManifests);

Expand All @@ -223,7 +243,8 @@ public void buildBakeRecipeSelectsHelmExecutableByVersion(
RoscoHelmConfigurationProperties helmConfigurationProperties =
new RoscoHelmConfigurationProperties();
HelmTemplateUtils helmTemplateUtils =
new HelmTemplateUtils(artifactDownloader, helmConfigurationProperties);
new HelmTemplateUtils(
artifactDownloader, Optional.empty(), artifactStoreConfig, helmConfigurationProperties);

HelmBakeManifestRequest request = new HelmBakeManifestRequest();
Artifact artifact = Artifact.builder().build();
Expand Down Expand Up @@ -256,7 +277,8 @@ public void buildBakeRecipeWithGitRepoArtifact(@TempDir Path tempDir) throws IOE
RoscoHelmConfigurationProperties helmConfigurationProperties =
new RoscoHelmConfigurationProperties();
HelmTemplateUtils helmTemplateUtils =
new HelmTemplateUtils(artifactDownloader, helmConfigurationProperties);
new HelmTemplateUtils(
artifactDownloader, Optional.empty(), artifactStoreConfig, helmConfigurationProperties);

HelmBakeManifestRequest request = new HelmBakeManifestRequest();

Expand Down Expand Up @@ -296,7 +318,8 @@ public void buildBakeRecipeWithGitRepoArtifactUsingHelmChartFilePath(@TempDir Pa
RoscoHelmConfigurationProperties helmConfigurationProperties =
new RoscoHelmConfigurationProperties();
HelmTemplateUtils helmTemplateUtils =
new HelmTemplateUtils(artifactDownloader, helmConfigurationProperties);
new HelmTemplateUtils(
artifactDownloader, Optional.empty(), artifactStoreConfig, helmConfigurationProperties);

HelmBakeManifestRequest request = new HelmBakeManifestRequest();

Expand Down Expand Up @@ -356,7 +379,8 @@ public void buildBakeRecipeIncludingHelmVersionsOptionsWithHelm3() throws IOExce
RoscoHelmConfigurationProperties helmConfigurationProperties =
new RoscoHelmConfigurationProperties();
HelmTemplateUtils helmTemplateUtils =
new HelmTemplateUtils(artifactDownloader, helmConfigurationProperties);
new HelmTemplateUtils(
artifactDownloader, Optional.empty(), artifactStoreConfig, helmConfigurationProperties);

HelmBakeManifestRequest request = new HelmBakeManifestRequest();
Artifact artifact = Artifact.builder().build();
Expand All @@ -381,7 +405,8 @@ public void buildBakeRecipeIncludingHelmVersionsOptionsWithHelm2() throws IOExce
RoscoHelmConfigurationProperties helmConfigurationProperties =
new RoscoHelmConfigurationProperties();
HelmTemplateUtils helmTemplateUtils =
new HelmTemplateUtils(artifactDownloader, helmConfigurationProperties);
new HelmTemplateUtils(
artifactDownloader, Optional.empty(), artifactStoreConfig, helmConfigurationProperties);

HelmBakeManifestRequest request = new HelmBakeManifestRequest();
Artifact artifact = Artifact.builder().build();
Expand All @@ -406,7 +431,8 @@ public void buildBakeRecipeIncludingCRDsWithHelm3() throws IOException {
RoscoHelmConfigurationProperties helmConfigurationProperties =
new RoscoHelmConfigurationProperties();
HelmTemplateUtils helmTemplateUtils =
new HelmTemplateUtils(artifactDownloader, helmConfigurationProperties);
new HelmTemplateUtils(
artifactDownloader, Optional.empty(), artifactStoreConfig, helmConfigurationProperties);

HelmBakeManifestRequest request = new HelmBakeManifestRequest();
Artifact artifact = Artifact.builder().build();
Expand All @@ -423,6 +449,54 @@ public void buildBakeRecipeIncludingCRDsWithHelm3() throws IOException {
}
}

@ParameterizedTest
@MethodSource("ensureOverrides")
public void ensureOverridesGetEvaluated(
String reference,
Map<String, Object> overrides,
String expected,
Boolean artifactStoreNull,
Boolean expandOverrides) {
ArtifactStore artifactStore = null;
if (!artifactStoreNull) {
artifactStore = mock(ArtifactStore.class);
when(artifactStore.get(any())).thenReturn(Artifact.builder().reference(reference).build());
}
RoscoHelmConfigurationProperties helmConfigurationProperties =
new RoscoHelmConfigurationProperties();

// do not use the artifactStoreConfig field as we are trying to test various
// values of expandOverrides
ArtifactStoreConfigurationProperties artifactStoreConfiguration =
new ArtifactStoreConfigurationProperties();
ArtifactStoreConfigurationProperties.HelmConfig helmConfig =
new ArtifactStoreConfigurationProperties.HelmConfig();
helmConfig.setExpandOverrides(expandOverrides);
artifactStoreConfiguration.setHelm(helmConfig);

HelmTemplateUtils helmTemplateUtils =
new HelmTemplateUtils(
artifactDownloader,
Optional.ofNullable(artifactStore),
artifactStoreConfiguration,
helmConfigurationProperties);
HelmBakeManifestRequest request = new HelmBakeManifestRequest();
request.setOutputName("output_name");
request.setTemplateRenderer(BakeManifestRequest.TemplateRenderer.HELM3);
request.setOverrides(overrides);
BakeRecipe recipe =
helmTemplateUtils.buildCommand(request, List.of(), Path.of("template_path"));

assertThat(recipe.getCommand().contains(expected)).isTrue();
}

private static Stream<Arguments> ensureOverrides() {
return Stream.of(
Arguments.of("test", Map.of("foo", "ref://bar/baz"), "foo=test", false, true),
Arguments.of("test", Map.of("foo", "ref://bar/baz"), "foo=ref://bar/baz", false, false),
Arguments.of("test", Map.of("foo", "ref://bar/baz"), "foo=ref://bar/baz", true, false));
}

@ParameterizedTest
@MethodSource("helmRendererArgsCRDs")
public void buildBakeRecipeNotIncludingCRDs(
Expand All @@ -432,7 +506,8 @@ public void buildBakeRecipeNotIncludingCRDs(
RoscoHelmConfigurationProperties helmConfigurationProperties =
new RoscoHelmConfigurationProperties();
HelmTemplateUtils helmTemplateUtils =
new HelmTemplateUtils(artifactDownloader, helmConfigurationProperties);
new HelmTemplateUtils(
artifactDownloader, Optional.empty(), artifactStoreConfig, helmConfigurationProperties);

HelmBakeManifestRequest request = new HelmBakeManifestRequest();
Artifact artifact = Artifact.builder().build();
Expand Down
Loading
Loading