Skip to content

Commit

Permalink
[k8s] Set resource limits and requests for Edge runtime components (#…
Browse files Browse the repository at this point in the history
…3666)

* Make agent config a configMap: Helm and Rust portion.

* Make agent config a configMap. csharp unit tests pass.

* Mounting agent config map as optional

* Make sure resources are not set if null in Values.

* Explain what the appsettings and its fields are used for.
  • Loading branch information
darobs authored Oct 15, 2020
1 parent 883f098 commit 1448111
Show file tree
Hide file tree
Showing 15 changed files with 656 additions and 56 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// Copyright (c) Microsoft. All rights reserved.

namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes
{
using System.Collections.Generic;
using System.Linq;
using k8s.Models;
using Microsoft.Azure.Devices.Edge.Agent.Kubernetes.EdgeDeployment.Service;
using Microsoft.Azure.Devices.Edge.Util;

/// <summary>
/// Application settings, derived from "appsetting_k8s.json" and environment
/// variables, loaded from autofac. These setting are used to allow edgeAgent
/// to properly construct proxy container settings and for agent to build itself.
/// </summary>
public class KubernetesApplicationSettings
{
// Location of "appsetting_k8s.json" in container
public string AgentConfigPath { get; set; }
// Name of configmap containing "appsetting_k8s.json"
public string AgentConfigMapName { get; set; }
// Name of volume to use when mounting configmap, used in PodSpec
public string AgentConfigVolume { get; set; }
// Proxy Image
public string ProxyImage { get; set; }
// Name of volume to use when mounting proxy's configmap
public string ProxyConfigVolume { get; set; }
// Name of configmap containing "config.yaml"
public string ProxyConfigMapName { get; set; }
// Location of "config.yaml" in proxy container
public string ProxyConfigPath { get; set; }
// Location of trustbundle in proxy container
public string ProxyTrustBundlePath { get; set; }
// Name of volume to use when mounting trustbundle
public string ProxyTrustBundleVolume { get; set; }
// Map of configmap containing trustbundle
public string ProxyTrustBundleConfigMapName { get; set; }
// Resource requirements for proxy.
public ResourceSettings ProxyResourceRequests { get; set; }
// Resource requirements for agent.
public ResourceSettings AgentResourceRequests { get; set; }

public Option<V1ResourceRequirements> GetProxyResourceRequirements() =>
Option.Maybe(this.ProxyResourceRequests).Map(rr => rr.ToResourceRequirements());
public Option<V1ResourceRequirements> GetAgentResourceRequirements() =>
Option.Maybe(this.AgentResourceRequests).Map(rr => rr.ToResourceRequirements());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
// Copyright (c) Microsoft. All rights reserved.

namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes
{
using System.Collections.Generic;
using System.Linq;
using k8s.Models;
using Microsoft.Azure.Devices.Edge.Util;

/// <summary>
/// K8s Resource requirements as read in by autofac
/// </summary>
public class ResourceSettings
{
public Dictionary<string, string> Limits { get; set; }
public Dictionary<string, string> Requests { get; set; }

public V1ResourceRequirements ToResourceRequirements()
{
Dictionary<string, ResourceQuantity> limits = Option.Maybe(this.Limits).Map(limitMap =>
limitMap.ToDictionary(pair => pair.Key, pair => new ResourceQuantity(pair.Value))).OrDefault();
Dictionary<string, ResourceQuantity> requests = Option.Maybe(this.Requests).Map(limitMap =>
limitMap.ToDictionary(pair => pair.Key, pair => new ResourceQuantity(pair.Value))).OrDefault();
return new V1ResourceRequirements(limits, requests);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ public class KubernetesDeploymentMapper : IKubernetesDeploymentMapper
readonly string proxyTrustBundlePath;
readonly string proxyTrustBundleVolumeName;
readonly string proxyTrustBundleConfigMapName;
readonly Option<V1ResourceRequirements> proxyResourceRequirements;
readonly Option<string> agentConfigMapName;
readonly Option<string> agentConfigPath;
readonly Option<string> agentConfigVolume;
readonly Option<V1ResourceRequirements> agentResourceRequirements;
readonly PortMapServiceType defaultServiceType;
readonly bool useMountSourceForVolumeName;
readonly Option<string> storageClassName;
Expand All @@ -52,6 +57,11 @@ public KubernetesDeploymentMapper(
string proxyTrustBundlePath,
string proxyTrustBundleVolumeName,
string proxyTrustBundleConfigMapName,
Option<V1ResourceRequirements> proxyResourceRequirements,
Option<string> agentConfigMapName,
Option<string> agentConfigPath,
Option<string> agentConfigVolume,
Option<V1ResourceRequirements> agentResourceRequirements,
PortMapServiceType defaultServiceType,
bool useMountSourceForVolumeName,
string storageClassName,
Expand All @@ -73,6 +83,11 @@ public KubernetesDeploymentMapper(
this.proxyTrustBundlePath = proxyTrustBundlePath;
this.proxyTrustBundleVolumeName = proxyTrustBundleVolumeName;
this.proxyTrustBundleConfigMapName = proxyTrustBundleConfigMapName;
this.proxyResourceRequirements = proxyResourceRequirements;
this.agentConfigMapName = agentConfigMapName;
this.agentConfigPath = agentConfigPath;
this.agentConfigVolume = agentConfigVolume;
this.agentResourceRequirements = agentResourceRequirements;
this.defaultServiceType = defaultServiceType;
this.useMountSourceForVolumeName = useMountSourceForVolumeName;
this.storageClassName = Option.Maybe(storageClassName);
Expand Down Expand Up @@ -180,7 +195,7 @@ V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule
{
List<V1EnvVar> env = this.CollectModuleEnv(module, identity);

(List<V1Volume> volumes, List<V1VolumeMount> volumeMounts) = this.CollectModuleVolumes(module);
(List<V1Volume> volumes, List<V1VolumeMount> volumeMounts) = this.CollectModuleVolumes(module, identity);

Option<V1SecurityContext> securityContext = module.Config.CreateOptions.HostConfig
.Filter(config => config.Privileged)
Expand All @@ -189,6 +204,8 @@ V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule
Option<List<V1ContainerPort>> exposedPorts = module.Config.CreateOptions.ExposedPorts
.Map(PortExtensions.GetContainerPorts);

Option<V1ResourceRequirements> resourceRequirements = this.PrepareModuleResourceRequirements(module, identity);

var container = new V1Container
{
Name = name,
Expand All @@ -197,7 +214,7 @@ V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule
VolumeMounts = volumeMounts,
SecurityContext = securityContext.OrDefault(),
Ports = exposedPorts.OrDefault(),
Resources = module.Config.CreateOptions.Resources.OrDefault(),
Resources = resourceRequirements.OrDefault(),
Command = module.Config.CreateOptions.Entrypoint.Map(list => list.ToList()).OrDefault(),
Args = module.Config.CreateOptions.Cmd.Map(list => list.ToList()).OrDefault(),
WorkingDir = module.Config.CreateOptions.WorkingDir.OrDefault(),
Expand Down Expand Up @@ -271,11 +288,25 @@ static IEnumerable<V1EnvVar> ParseEnv(IReadOnlyList<string> env) =>
.Where(keyValue => keyValue.Length == 2)
.Select(keyValue => new V1EnvVar(keyValue[0], keyValue[1]));

(List<V1Volume>, List<V1VolumeMount>) CollectModuleVolumes(KubernetesModule module)
(List<V1Volume>, List<V1VolumeMount>) CollectModuleVolumes(KubernetesModule module, IModuleIdentity identity)
{
var volumeList = new List<V1Volume>();
var volumeMountList = new List<V1VolumeMount>();

if (string.Equals(identity.ModuleId, CoreConstants.EdgeAgentModuleIdentityName))
{
// If iotedged is new enough, edgeAgent now has a config map which needs
// to be mounted.
if (this.agentConfigMapName.HasValue &&
this.agentConfigPath.HasValue &&
this.agentConfigVolume.HasValue)
{
var agentConfigVolume = this.agentConfigVolume.OrDefault();
volumeList.Add(new V1Volume { Name = agentConfigVolume, ConfigMap = new V1ConfigMapVolumeSource(name: this.agentConfigMapName.OrDefault(), optional: true) });
volumeMountList.Add(new V1VolumeMount { MountPath = this.agentConfigPath.OrDefault(), Name = agentConfigVolume, ReadOnlyProperty = true });
}
}

// collect volumes and volume mounts from HostConfig.Binds section
var binds = module.Config.CreateOptions.HostConfig
.FlatMap(config => Option.Maybe(config.Binds))
Expand Down Expand Up @@ -328,6 +359,19 @@ static IEnumerable<V1EnvVar> ParseEnv(IReadOnlyList<string> env) =>
return (volumeList, volumeMountList);
}

Option<V1ResourceRequirements> PrepareModuleResourceRequirements(KubernetesModule module, IModuleIdentity identity)
{
// All modules get their resource requirements from createOptions,
// but Agent has default resources to guarantee QoS if the user didn't set it.
if (!module.Config.CreateOptions.Resources.HasValue &&
string.Equals(identity.ModuleId, CoreConstants.EdgeAgentModuleIdentityName))
{
return this.agentResourceRequirements;
}

return module.Config.CreateOptions.Resources;
}

(V1Container, List<V1Volume>) PrepareProxyContainer(KubernetesModule module)
{
var env = new List<V1EnvVar>
Expand All @@ -337,7 +381,7 @@ static IEnumerable<V1EnvVar> ParseEnv(IReadOnlyList<string> env) =>

var volumeMounts = new List<V1VolumeMount>
{
new V1VolumeMount { MountPath = this.proxyConfigPath, Name = this.proxyConfigVolumeName },
new V1VolumeMount { MountPath = this.proxyConfigPath, Name = this.proxyConfigVolumeName, ReadOnlyProperty = true },
new V1VolumeMount { MountPath = this.proxyTrustBundlePath, Name = this.proxyTrustBundleVolumeName }
};

Expand All @@ -347,11 +391,14 @@ static IEnumerable<V1EnvVar> ParseEnv(IReadOnlyList<string> env) =>
new V1Volume { Name = this.proxyTrustBundleVolumeName, ConfigMap = new V1ConfigMapVolumeSource(name: this.proxyTrustBundleConfigMapName) }
};

var resources = this.proxyResourceRequirements.OrDefault();

var container = new V1Container
{
Name = "proxy",
Env = env,
Image = this.proxyImage,
Resources = resources,
VolumeMounts = volumeMounts
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Service
public class Program
{
const string ConfigFileName = "appsettings_agent.json";
const string K8sConfigFileName = "/etc/edgeAgent/appsettings_k8s.json";
const string DefaultLocalConfigFilePath = "config.json";
const string EdgeAgentStorageFolder = "edgeAgent";
const string EdgeAgentStorageBackupFolder = "edgeAgent_backup";
Expand Down Expand Up @@ -181,14 +182,14 @@ public static async Task<int> MainAsync(IConfiguration configuration)
iothubHostname = configuration.GetValue<string>(Constants.IotHubHostnameVariableName);
builder.RegisterInstance(new NullMetricsProvider() as IMetricsProvider);
deviceId = configuration.GetValue<string>(Constants.DeviceIdVariableName);
string proxyImage = configuration.GetValue<string>(K8sConstants.ProxyImageEnvKey);
// Get additional k8s configuration from the configmap and environment.
IConfigurationRoot k8sConfiguration = new ConfigurationBuilder()
.AddJsonFile(K8sConfigFileName, true)
.AddEnvironmentVariables()
.Build();
// k8s options
KubernetesApplicationSettings k8sSettings = k8sConfiguration.Get<KubernetesApplicationSettings>();
Option<string> proxyImagePullSecretName = Option.Maybe(configuration.GetValue<string>(K8sConstants.ProxyImagePullSecretNameEnvKey));
string proxyConfigPath = configuration.GetValue<string>(K8sConstants.ProxyConfigPathEnvKey);
string proxyConfigVolumeName = configuration.GetValue<string>(K8sConstants.ProxyConfigVolumeEnvKey);
string proxyConfigMapName = configuration.GetValue<string>(K8sConstants.ProxyConfigMapNameEnvKey);
string proxyTrustBundlePath = configuration.GetValue<string>(K8sConstants.ProxyTrustBundlePathEnvKey);
string proxyTrustBundleVolumeName = configuration.GetValue<string>(K8sConstants.ProxyTrustBundleVolumeEnvKey);
string proxyTrustBundleConfigMapName = configuration.GetValue<string>(K8sConstants.ProxyTrustBundleConfigMapEnvKey);
PortMapServiceType mappedServiceDefault = GetDefaultServiceType(configuration);
bool enableServiceCallTracing = configuration.GetValue<bool>(K8sConstants.EnableK8sServiceCallTracingName);
bool useMountSourceForVolumeName = configuration.GetValue<bool>(K8sConstants.UseMountSourceForVolumeNameKey, false);
Expand All @@ -209,14 +210,8 @@ public static async Task<int> MainAsync(IConfiguration configuration)
iothubHostname,
deviceId,
edgeDeviceHostName,
proxyImage,
k8sSettings,
proxyImagePullSecretName,
proxyConfigPath,
proxyConfigVolumeName,
proxyConfigMapName,
proxyTrustBundlePath,
proxyTrustBundleVolumeName,
proxyTrustBundleConfigMapName,
apiVersion,
deviceNamespace,
new Uri(managementUri),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,8 @@ public class KubernetesModule : Module
static readonly TimeSpan SystemInfoTimeout = TimeSpan.FromSeconds(120);
readonly ResourceName resourceName;
readonly string edgeDeviceHostName;
readonly string proxyImage;
readonly KubernetesApplicationSettings k8sSettings;
readonly Option<string> proxyImagePullSecretName;
readonly string proxyConfigPath;
readonly string proxyConfigVolumeName;
readonly string proxyConfigMapName;
readonly string proxyTrustBundlePath;
readonly string proxyTrustBundleVolumeName;
readonly string proxyTrustBundleConfigMapName;
readonly string apiVersion;
readonly string deviceNamespace;
readonly string deviceSelector;
Expand All @@ -69,14 +63,8 @@ public KubernetesModule(
string iotHubHostname,
string deviceId,
string edgeDeviceHostName,
string proxyImage,
KubernetesApplicationSettings k8sSettings,
Option<string> proxyImagePullSecretName,
string proxyConfigPath,
string proxyConfigVolumeName,
string proxyConfigMapName,
string proxyTrustBundlePath,
string proxyTrustBundleVolumeName,
string proxyTrustBundleConfigMapName,
string apiVersion,
string deviceNamespace,
Uri managementUri,
Expand All @@ -99,14 +87,15 @@ public KubernetesModule(
{
this.resourceName = new ResourceName(iotHubHostname, deviceId);
this.edgeDeviceHostName = Preconditions.CheckNonWhiteSpace(edgeDeviceHostName, nameof(edgeDeviceHostName));
this.proxyImage = Preconditions.CheckNonWhiteSpace(proxyImage, nameof(proxyImage));
this.k8sSettings = Preconditions.CheckNotNull(k8sSettings, nameof(k8sSettings));
Preconditions.CheckNonWhiteSpace(k8sSettings.ProxyImage, nameof(k8sSettings.ProxyImage));
Preconditions.CheckNonWhiteSpace(k8sSettings.ProxyConfigPath, nameof(k8sSettings.ProxyConfigPath));
Preconditions.CheckNonWhiteSpace(k8sSettings.ProxyConfigVolume, nameof(k8sSettings.ProxyConfigVolume));
Preconditions.CheckNonWhiteSpace(k8sSettings.ProxyConfigMapName, nameof(k8sSettings.ProxyConfigMapName));
Preconditions.CheckNonWhiteSpace(k8sSettings.ProxyTrustBundlePath, nameof(k8sSettings.ProxyTrustBundlePath));
Preconditions.CheckNonWhiteSpace(k8sSettings.ProxyTrustBundleVolume, nameof(k8sSettings.ProxyTrustBundleVolume));
Preconditions.CheckNonWhiteSpace(k8sSettings.ProxyTrustBundleConfigMapName, nameof(k8sSettings.ProxyTrustBundleConfigMapName));
this.proxyImagePullSecretName = proxyImagePullSecretName;
this.proxyConfigPath = Preconditions.CheckNonWhiteSpace(proxyConfigPath, nameof(proxyConfigPath));
this.proxyConfigVolumeName = Preconditions.CheckNonWhiteSpace(proxyConfigVolumeName, nameof(proxyConfigVolumeName));
this.proxyConfigMapName = Preconditions.CheckNonWhiteSpace(proxyConfigMapName, nameof(proxyConfigMapName));
this.proxyTrustBundlePath = Preconditions.CheckNonWhiteSpace(proxyTrustBundlePath, nameof(proxyTrustBundlePath));
this.proxyTrustBundleVolumeName = Preconditions.CheckNonWhiteSpace(proxyTrustBundleVolumeName, nameof(proxyTrustBundleVolumeName));
this.proxyTrustBundleConfigMapName = Preconditions.CheckNonWhiteSpace(proxyTrustBundleConfigMapName, nameof(proxyTrustBundleConfigMapName));
this.apiVersion = Preconditions.CheckNonWhiteSpace(apiVersion, nameof(apiVersion));
this.deviceSelector = $"{Constants.K8sEdgeDeviceLabel}={KubeUtils.SanitizeLabelValue(this.resourceName.DeviceId)}";
this.deviceNamespace = Preconditions.CheckNonWhiteSpace(deviceNamespace, nameof(deviceNamespace));
Expand Down Expand Up @@ -247,14 +236,19 @@ protected override void Load(ContainerBuilder builder)
c => new KubernetesDeploymentMapper(
this.deviceNamespace,
this.edgeDeviceHostName,
this.proxyImage,
this.k8sSettings.ProxyImage,
this.proxyImagePullSecretName,
this.proxyConfigPath,
this.proxyConfigVolumeName,
this.proxyConfigMapName,
this.proxyTrustBundlePath,
this.proxyTrustBundleVolumeName,
this.proxyTrustBundleConfigMapName,
this.k8sSettings.ProxyConfigPath,
this.k8sSettings.ProxyConfigVolume,
this.k8sSettings.ProxyConfigMapName,
this.k8sSettings.ProxyTrustBundlePath,
this.k8sSettings.ProxyTrustBundleVolume,
this.k8sSettings.ProxyTrustBundleConfigMapName,
this.k8sSettings.GetProxyResourceRequirements(),
Option.Maybe(this.k8sSettings.AgentConfigMapName),
Option.Maybe(this.k8sSettings.AgentConfigPath),
Option.Maybe(this.k8sSettings.AgentConfigVolume),
this.k8sSettings.GetAgentResourceRequirements(),
this.defaultMapServiceType,
this.useMountSourceForVolumeName,
this.storageClassName,
Expand Down
Loading

0 comments on commit 1448111

Please sign in to comment.