Skip to content

Commit

Permalink
Helm: Fix handling of Knative probes
Browse files Browse the repository at this point in the history
This issue happens when having multiple extensions at the classpath.
Relates quarkiverse/quarkus-helm#222
  • Loading branch information
Sgitario committed Apr 4, 2023
1 parent e9f1120 commit 25ebbe5
Show file tree
Hide file tree
Showing 9 changed files with 334 additions and 44 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import static io.dekorate.helm.util.YamlExpressionParserUtils.SEPARATOR_QUOTES;
import static io.dekorate.helm.util.YamlExpressionParserUtils.SEPARATOR_TOKEN;
import static io.dekorate.helm.util.YamlExpressionParserUtils.START_EXPRESSION_TOKEN;
import static io.dekorate.helm.util.YamlExpressionParserUtils.read;
import static io.dekorate.helm.util.YamlExpressionParserUtils.readAndSet;
import static io.dekorate.helm.util.YamlExpressionParserUtils.set;

Expand Down Expand Up @@ -146,23 +147,23 @@ public void onClosed() {
*/
public Map<String, String> writeHelmFiles(Project project,
HelmChartConfig helmConfig,
List<ConfigReference> configReferencesFromConfig,
List<ConfigReference> configReferencesFromDecorators,
List<ConfigReference> valueReferencesFromUser,
List<ConfigReference> valueReferencesFromDecorators,
Path inputDir,
Path outputDir,
Collection<File> generatedFiles) {
Map<String, String> artifacts = new HashMap<>();
if (helmConfig.isEnabled()) {
validateHelmConfig(helmConfig);
List<ConfigReference> valuesReferences = mergeValuesReferencesFromDecorators(helmConfig, configReferencesFromConfig,
configReferencesFromDecorators);

try {
LOGGER.info(String.format("Creating Helm Chart \"%s\"", helmConfig.getName()));
ValuesHolder values = populateValues(helmConfig, valuesReferences);
artifacts.putAll(processTemplates(helmConfig, inputDir, outputDir, generatedFiles, valuesReferences, values));
ValuesHolder values = populateValuesFromConfig(helmConfig);
List<Map<Object, Object>> resources = populateValuesFromConfigReferences(helmConfig, generatedFiles, values,
valueReferencesFromUser, valueReferencesFromDecorators);
artifacts.putAll(processTemplates(helmConfig, inputDir, outputDir, resources));
artifacts.putAll(createChartYaml(helmConfig, project, inputDir, outputDir));
artifacts.putAll(createValuesYaml(helmConfig, valuesReferences, inputDir, outputDir, values));
artifacts.putAll(createValuesYaml(helmConfig, inputDir, outputDir, values));

// To follow Helm file structure standards:
artifacts.putAll(createEmptyChartFolder(helmConfig, outputDir));
Expand Down Expand Up @@ -309,26 +310,65 @@ private Map<String, String> createEmptyChartFolder(HelmChartConfig helmConfig, P
return Collections.singletonMap(emptyChartsDir.toString(), EMPTY);
}

private List<ConfigReference> mergeValuesReferencesFromDecorators(HelmChartConfig helmConfig,
List<ConfigReference> configReferencesFromConfig, List<ConfigReference> configReferencesFromDecorators) {
List<ConfigReference> configReferences = new LinkedList<>();
// From user
configReferences.addAll(configReferencesFromConfig);
// From if statements: these are boolean values
for (AddIfStatement addIfStatement : helmConfig.getAddIfStatements()) {
configReferences.add(new ConfigReference.Builder(deductProperty(helmConfig, addIfStatement.getProperty()), new String[0])
.withDescription(addIfStatement.getDescription())
.withValue(addIfStatement.getWithDefaultValue())
.build());
}
// From decorators
configReferences.addAll(configReferencesFromDecorators);
private List<Map<Object, Object>> populateValuesFromConfigReferences(io.dekorate.helm.config.HelmChartConfig helmConfig,
Collection<File> generatedFiles,
ValuesHolder values,
List<ConfigReference> valuesReferencesFromUser,
List<ConfigReference> valuesReferencesFromDecorators) throws IOException {
List<Map<Object, Object>> allResources = new LinkedList<>();
for (File generatedFile : generatedFiles) {
if (!generatedFile.getName().toLowerCase().matches(YAML_REG_EXP)) {
continue;
}

return configReferences;
}
// Read helm expression parsers
YamlExpressionParser parser = YamlPath.from(new FileInputStream(generatedFile));

// Seen lookup by default values.yaml file.
Map<String, Object> seen = new HashMap<>();

// Merge all values references in order: first the users' and then the decorators'.
List<ConfigReference> valuesReferences = new ArrayList<>();
valuesReferences.addAll(valuesReferencesFromUser);
valuesReferences.addAll(valuesReferencesFromDecorators);

// First, process the non-environmental properties
for (ConfigReference valueReference : valuesReferences) {
if (!valueIsEnvironmentProperty(valueReference)) {
String valueReferenceProperty = deductProperty(helmConfig, valueReference.getProperty());

processValueReference(valueReferenceProperty, valueReference.getValue(), valueReference, values, parser,
seen, valuesReferencesFromUser.contains(valueReference));
}
}

// Next, process the environmental properties, so we can decide if it's a property coming from values.yaml or not.
for (ConfigReference valueReference : valuesReferences) {
if (valueIsEnvironmentProperty(valueReference)) {
String valueReferenceProperty = deductProperty(helmConfig, valueReference.getProperty());
Object valueReferenceValue = valueReference.getValue();
String environmentProperty = getEnvironmentPropertyName(valueReference);

// Try to find the value from the current values
Map<String, ValuesHolder.HelmValueHolder> current = values.get(valueReference.getProfile());
for (Map.Entry<String, ValuesHolder.HelmValueHolder> currentValue : current.entrySet()) {
if (currentValue.getKey().endsWith(environmentProperty)) {
// found, we use this value instead of generating an additional envs.xxx=yyy property
valueReferenceProperty = currentValue.getKey();
valueReferenceValue = currentValue.getValue();
break;
}
}

processValueReference(valueReferenceProperty, valueReferenceValue, valueReference, values, parser, seen,
valuesReferencesFromUser.contains(valueReference));
}
}

private boolean valueHasPath(ConfigReference valueReference) {
return valueReference.getPaths() != null && valueReference.getPaths().length > 0;
allResources.addAll(parser.getResources());
}

return allResources;
}

private ConfigReference toConfigReference(ValueReference valueReference) {
Expand All @@ -344,8 +384,7 @@ private ConfigReference toConfigReference(ValueReference valueReference) {
.build();
}

private Map<String, String> createValuesYaml(HelmChartConfig helmConfig, List<ConfigReference> configReferences,
Path inputDir, Path outputDir,
private Map<String, String> createValuesYaml(HelmChartConfig helmConfig, Path inputDir, Path outputDir,
ValuesHolder valuesHolder) throws IOException {

Map<String, ValuesHolder.HelmValueHolder> prodValues = valuesHolder.getProdValues();
Expand Down Expand Up @@ -459,12 +498,11 @@ private String getVersion(HelmChartConfig helmConfig, Project project) {
}

private Map<String, String> processTemplates(HelmChartConfig helmConfig, Path inputDir, Path outputDir,
Collection<File> generatedFiles, List<ConfigReference> valuesReferences, ValuesHolder values) throws IOException {
List<Map<Object, Object>> resources) throws IOException {

Map<String, String> templates = new HashMap<>();
Path templatesDir = getChartOutputDir(helmConfig, outputDir).resolve(TEMPLATES);
Files.createDirectories(templatesDir);
List<Map<Object, Object>> resources = replaceValuesInYamls(helmConfig, generatedFiles, valuesReferences, values);
Map<String, String> functionsByResource = processUserDefinedTemplates(inputDir, templates, templatesDir);
// Split yamls in separated files by kind
for (Map<Object, Object> resource : resources) {
Expand Down Expand Up @@ -569,22 +607,9 @@ private Map<String, String> processUserDefinedTemplates(Path inputDir, Map<Strin
return functionsByResource;
}

private ValuesHolder populateValues(io.dekorate.helm.config.HelmChartConfig helmConfig,
List<ConfigReference> valuesReferences) {
private ValuesHolder populateValuesFromConfig(io.dekorate.helm.config.HelmChartConfig helmConfig) {
ValuesHolder values = new ValuesHolder();

// Populate user prod values without expression from properties
for (ConfigReference value : valuesReferences) {
if (!valueHasPath(value)) {
if (value.getValue() == null) {
throw new RuntimeException("The value mapping for " + value.getProperty() + " does not have "
+ "either a path or a default value. ");
}

values.put(deductProperty(helmConfig, value.getProperty()), value);
}
}

// Populate expressions from conditions
for (io.dekorate.helm.config.HelmDependency dependency : helmConfig.getDependencies()) {
if (Strings.isNotNullOrEmpty(dependency.getCondition())) {
Expand All @@ -595,6 +620,16 @@ private ValuesHolder populateValues(io.dekorate.helm.config.HelmChartConfig helm
}
}

// Populate if statements expressions
for (AddIfStatement addIfStatement : helmConfig.getAddIfStatements()) {
ConfigReference configReference = new ConfigReference.Builder(
deductProperty(helmConfig, addIfStatement.getProperty()), new String[0])
.withDescription(addIfStatement.getDescription())
.withValue(addIfStatement.getWithDefaultValue())
.build();
values.put(deductProperty(helmConfig, addIfStatement.getProperty()), configReference);
}

return values;
}

Expand Down Expand Up @@ -698,6 +733,46 @@ private void processValueReference(String property, Object value, ConfigReferenc
}
}

private void processValueReference(String property, Object value, ConfigReference valueReference, ValuesHolder values,
YamlExpressionParser parser, Map<String, Object> seen, boolean isUserReference) {

String profile = valueReference.getProfile();
String expression = Optional.ofNullable(valueReference.getExpression())
.filter(Strings::isNotNullOrEmpty)
.orElse(VALUES_START_TAG + property + VALUES_END_TAG);

if (valueReference.getPaths() != null && valueReference.getPaths().length > 0) {
for (String path : valueReference.getPaths()) {
Object found = seen.get(property);
if (found == null) {
found = read(parser, path);
}

Object actualValue = null;
if (isUserReference) {
// if the value is coming from the user, we use the provided value
actualValue = Optional.ofNullable(value).orElse(found);
} else {
// if the value is coming from one decorator, we use the found value
actualValue = Optional.ofNullable(found).orElse(value);
}

if (actualValue != null) {
set(parser, path, expression);
values.putIfAbsent(property, valueReference, actualValue, profile);
if (Strings.isNullOrEmpty(profile)) {
seen.putIfAbsent(property, actualValue);
}
}
}
} else {
values.putIfAbsent(property, valueReference, value, profile);
if (Strings.isNullOrEmpty(profile)) {
seen.putIfAbsent(property, value);
}
}
}

private Map<String, String> createChartYaml(HelmChartConfig helmConfig, Project project, Path inputDir, Path outputDir)
throws IOException {
final Chart chart = new Chart();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ public static void set(YamlExpressionParser parser, String path, String expressi
parser.write(path, adaptExpression(expression));
}

public static Object read(YamlExpressionParser parser, String path) {
Set<Object> found = parser.read(path);
return found.stream().findFirst().orElse(null);
}

public static Object readAndSet(YamlExpressionParser parser, String path, String expression) {
Set<Object> found = parser.readAndReplace(path, adaptExpression(expression));
return found.stream().findFirst().orElse(null);
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/io/dekorate/ConfigReference.java
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public class ConfigReference {
private final Set<String> enumValues;
private final boolean required;

private ConfigReference(String property, String[] paths, String description, Object value, String expression,
protected ConfigReference(String property, String[] paths, String description, Object value, String expression,
String profile, Integer minimum, Integer maximum, String pattern, Set<String> enumValues, boolean required) {
this.property = property;
this.paths = paths;
Expand Down
92 changes: 92 additions & 0 deletions tests/issue-knative-probes-helm/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<artifactId>dekorate-tests</artifactId>
<groupId>io.dekorate</groupId>
<version>3.5-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

<groupId>io.dekorate</groupId>
<artifactId>issue-knative-probes-helm</artifactId>
<name>Dekorate :: Tests :: Annotations :: Helm :: Generate Knative probes</name>
<description></description>

<properties>
<property.secret.name>my-secret</property.secret.name>
</properties>

<dependencies>
<dependency>
<groupId>io.dekorate</groupId>
<artifactId>kubernetes-annotations</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.dekorate</groupId>
<artifactId>knative-annotations</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.dekorate</groupId>
<artifactId>helm-annotations</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.dekorate</groupId>
<artifactId>dekorate-spring-boot</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<version>${version.spring-boot}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
<version>${version.spring-boot}</version>
</dependency>

<!-- Testing -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>${version.junit-jupiter}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>${version.junit-jupiter}</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<inherited>true</inherited>
<configuration>
<useSystemClassLoader>false</useSystemClassLoader>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${version.spring-boot}</version>
</plugin>
</plugins>
</build>
</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/**
* Copyright 2018 The original authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.dekorate.knative.helm;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
public class DemoApplication {

public static void main(String[] args) {
SpringApplication.run(DemoApplication.class, args);
}

}
Loading

0 comments on commit 25ebbe5

Please sign in to comment.