diff --git a/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/LinuxUtilizationProvider.cs b/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/LinuxUtilizationProvider.cs index 0c3d4124e17..611b96f4f1d 100644 --- a/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/LinuxUtilizationProvider.cs +++ b/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/LinuxUtilizationProvider.cs @@ -37,7 +37,7 @@ internal sealed class LinuxUtilizationProvider : ISnapshotProvider private DateTimeOffset _refreshAfterMemory; private double _cpuPercentage = double.NaN; private double _lastCpuCoresUsed = double.NaN; - private double _memoryPercentage; + private ulong _memoryUsage; private long _previousCgroupCpuTime; private long _previousHostCpuTime; private long _previousCgroupCpuPeriodCounter; @@ -116,12 +116,18 @@ public LinuxUtilizationProvider(IOptions options, ILi _ = meter.CreateObservableGauge( name: ResourceUtilizationInstruments.ContainerMemoryLimitUtilization, - observeValues: () => GetMeasurementWithRetry(MemoryUtilization), + observeValues: () => GetMeasurementWithRetry(MemoryPercentage), unit: "1"); + _ = meter.CreateObservableUpDownCounter( + name: ResourceUtilizationInstruments.ContainerMemoryUsage, + observeValues: () => GetMeasurementWithRetry(() => (long)MemoryUsage()), + unit: "By", + description: "Memory usage of the container."); + _ = meter.CreateObservableGauge( name: ResourceUtilizationInstruments.ProcessMemoryUtilization, - observeValues: () => GetMeasurementWithRetry(MemoryUtilization), + observeValues: () => GetMeasurementWithRetry(MemoryPercentage), unit: "1"); // cpuRequest is a CPU request (aka guaranteed number of CPU units) for pod, for host its 1 core @@ -216,7 +222,7 @@ public double CpuUtilization() return _cpuPercentage; } - public double MemoryUtilization() + public ulong MemoryUsage() { DateTimeOffset now = _timeProvider.GetUtcNow(); @@ -224,26 +230,24 @@ public double MemoryUtilization() { if (now < _refreshAfterMemory) { - return _memoryPercentage; + return _memoryUsage; } } - ulong memoryUsed = _parser.GetMemoryUsageInBytes(); + ulong memoryUsage = _parser.GetMemoryUsageInBytes(); lock (_memoryLocker) { if (now >= _refreshAfterMemory) { - double memoryPercentage = Math.Min(One, (double)memoryUsed / _memoryLimit); - - _memoryPercentage = memoryPercentage; + _memoryUsage = memoryUsage; _refreshAfterMemory = now.Add(_memoryRefreshInterval); } } - _logger.MemoryUsageData(memoryUsed, _memoryLimit, _memoryPercentage); + _logger.MemoryUsageData(_memoryUsage); - return _memoryPercentage; + return _memoryUsage; } /// @@ -264,14 +268,24 @@ public Snapshot GetSnapshot() memoryUsageInBytes: memoryUsed); } - private Measurement[] GetMeasurementWithRetry(Func func) + private double MemoryPercentage() + { + ulong memoryUsage = MemoryUsage(); + double memoryPercentage = Math.Min(One, (double)memoryUsage / _memoryLimit); + + _logger.MemoryPercentageData(memoryUsage, _memoryLimit, memoryPercentage); + return memoryPercentage; + } + + private Measurement[] GetMeasurementWithRetry(Func func) + where T : struct { - if (!TryGetValueWithRetry(func, out double value)) + if (!TryGetValueWithRetry(func, out T value)) { - return Array.Empty>(); + return Array.Empty>(); } - return new[] { new Measurement(value) }; + return new[] { new Measurement(value) }; } private bool TryGetValueWithRetry(Func func, out T value) diff --git a/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/Log.cs b/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/Log.cs index 209a495e844..c021f48bb1b 100644 --- a/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/Log.cs +++ b/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/Log.cs @@ -24,7 +24,7 @@ public static partial void CpuUsageData( [LoggerMessage(2, LogLevel.Debug, "Computed memory usage with MemoryUsedInBytes = {memoryUsed}, MemoryLimit = {memoryLimit}, MemoryPercentage = {memoryPercentage}.")] - public static partial void MemoryUsageData( + public static partial void MemoryPercentageData( this ILogger logger, ulong memoryUsed, double memoryLimit, @@ -55,4 +55,10 @@ public static partial void CpuUsageDataV2( public static partial void HandleDiskStatsException( this ILogger logger, string errorMessage); + + [LoggerMessage(6, LogLevel.Debug, + "Computed memory usage = {memoryUsed}.")] + public static partial void MemoryUsageData( + this ILogger logger, + ulong memoryUsed); } diff --git a/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/Log.cs b/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/Log.cs index fdc0d17fe44..e8d8411925d 100644 --- a/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/Log.cs +++ b/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/Log.cs @@ -26,12 +26,11 @@ public static partial void CpuUsageData( double cpuPercentage); [LoggerMessage(4, LogLevel.Debug, - "Computed memory usage with CurrentMemoryUsage = {currentMemoryUsage}, TotalMemory = {totalMemory}, MemoryPercentage = {memoryPercentage}.")] - public static partial void MemoryUsageData( + "Computed memory usage for container: CurrentMemoryUsage = {currentMemoryUsage}, TotalMemory = {totalMemory}")] + public static partial void ContainerMemoryUsageData( this ILogger logger, ulong currentMemoryUsage, - double totalMemory, - double memoryPercentage); + double totalMemory); #pragma warning disable S103 // Lines should not be too long [LoggerMessage(5, LogLevel.Debug, "Computed CPU usage with CpuUsageKernelTicks = {cpuUsageKernelTicks}, CpuUsageUserTicks = {cpuUsageUserTicks}, OldCpuUsageTicks = {oldCpuUsageTicks}, TimeTickDelta = {timeTickDelta}, CpuUnits = {cpuUnits}, CpuPercentage = {cpuPercentage}.")] @@ -60,4 +59,12 @@ public static partial void DiskIoPerfCounterException( this ILogger logger, string counterName, string errorMessage); + + [LoggerMessage(8, LogLevel.Debug, + "Computed memory usage for current process: ProcessMemoryUsage = {processMemoryUsage}, TotalMemory = {totalMemory}, MemoryPercentage = {memoryPercentage}")] + public static partial void ProcessMemoryPercentageData( + this ILogger logger, + ulong processMemoryUsage, + double totalMemory, + double memoryPercentage); } diff --git a/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/WindowsContainerSnapshotProvider.cs b/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/WindowsContainerSnapshotProvider.cs index ca6ceaff8bd..ce10cad0471 100644 --- a/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/WindowsContainerSnapshotProvider.cs +++ b/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/WindowsContainerSnapshotProvider.cs @@ -29,6 +29,7 @@ internal sealed class WindowsContainerSnapshotProvider : ISnapshotProvider private readonly object _cpuLocker = new(); private readonly object _memoryLocker = new(); + private readonly object _processMemoryLocker = new(); private readonly TimeProvider _timeProvider; private readonly IProcessInfo _processInfo; private readonly ILogger _logger; @@ -42,8 +43,10 @@ internal sealed class WindowsContainerSnapshotProvider : ISnapshotProvider private long _oldCpuTimeTicks; private DateTimeOffset _refreshAfterCpu; private DateTimeOffset _refreshAfterMemory; + private DateTimeOffset _refreshAfterProcessMemory; private double _cpuPercentage = double.NaN; - private double _memoryPercentage; + private ulong _memoryUsage; + private double _processMemoryPercentage; public SystemResources Resources { get; } @@ -107,6 +110,7 @@ internal WindowsContainerSnapshotProvider( _memoryRefreshInterval = options.MemoryConsumptionRefreshInterval; _refreshAfterCpu = _timeProvider.GetUtcNow(); _refreshAfterMemory = _timeProvider.GetUtcNow(); + _refreshAfterProcessMemory = _timeProvider.GetUtcNow(); #pragma warning disable CA2000 // Dispose objects before losing scope // We don't dispose the meter because IMeterFactory handles that @@ -116,13 +120,34 @@ internal WindowsContainerSnapshotProvider( #pragma warning restore CA2000 // Dispose objects before losing scope // Container based metrics: - _ = meter.CreateObservableCounter(name: ResourceUtilizationInstruments.ContainerCpuTime, observeValues: GetCpuTime, unit: "s", description: "CPU time used by the container."); - _ = meter.CreateObservableGauge(name: ResourceUtilizationInstruments.ContainerCpuLimitUtilization, observeValue: CpuPercentage); - _ = meter.CreateObservableGauge(name: ResourceUtilizationInstruments.ContainerMemoryLimitUtilization, observeValue: () => MemoryPercentage(() => _processInfo.GetMemoryUsage())); + _ = meter.CreateObservableCounter( + name: ResourceUtilizationInstruments.ContainerCpuTime, + observeValues: GetCpuTime, + unit: "s", + description: "CPU time used by the container."); + + _ = meter.CreateObservableGauge( + name: ResourceUtilizationInstruments.ContainerCpuLimitUtilization, + observeValue: CpuPercentage); + + _ = meter.CreateObservableGauge( + name: ResourceUtilizationInstruments.ContainerMemoryLimitUtilization, + observeValue: () => Math.Min(_metricValueMultiplier, MemoryUsage() / _memoryLimit * _metricValueMultiplier)); + + _ = meter.CreateObservableUpDownCounter( + name: ResourceUtilizationInstruments.ContainerMemoryUsage, + observeValue: () => (long)MemoryUsage(), + unit: "By", + description: "Memory usage of the container."); // Process based metrics: - _ = meter.CreateObservableGauge(name: ResourceUtilizationInstruments.ProcessCpuUtilization, observeValue: CpuPercentage); - _ = meter.CreateObservableGauge(name: ResourceUtilizationInstruments.ProcessMemoryUtilization, observeValue: () => MemoryPercentage(() => _processInfo.GetCurrentProcessMemoryUsage())); + _ = meter.CreateObservableGauge( + name: ResourceUtilizationInstruments.ProcessCpuUtilization, + observeValue: CpuPercentage); + + _ = meter.CreateObservableGauge( + name: ResourceUtilizationInstruments.ProcessMemoryUtilization, + observeValue: ProcessMemoryPercentage); } public Snapshot GetSnapshot() @@ -185,7 +210,35 @@ private ulong GetMemoryLimit(IJobHandle jobHandle) return memoryLimitInBytes; } - private double MemoryPercentage(Func getMemoryUsage) + private double ProcessMemoryPercentage() + { + DateTimeOffset now = _timeProvider.GetUtcNow(); + + lock (_processMemoryLocker) + { + if (now < _refreshAfterProcessMemory) + { + return _processMemoryPercentage; + } + } + + ulong processMemoryUsage = _processInfo.GetCurrentProcessMemoryUsage(); + + lock (_processMemoryLocker) + { + if (now >= _refreshAfterProcessMemory) + { + _processMemoryPercentage = Math.Min(_metricValueMultiplier, processMemoryUsage / _memoryLimit * _metricValueMultiplier); + _refreshAfterProcessMemory = now.Add(_memoryRefreshInterval); + + _logger.ProcessMemoryPercentageData(processMemoryUsage, _memoryLimit, _processMemoryPercentage); + } + + return _processMemoryPercentage; + } + } + + private ulong MemoryUsage() { DateTimeOffset now = _timeProvider.GetUtcNow(); @@ -193,24 +246,22 @@ private double MemoryPercentage(Func getMemoryUsage) { if (now < _refreshAfterMemory) { - return _memoryPercentage; + return _memoryUsage; } } - ulong memoryUsage = getMemoryUsage(); + ulong memoryUsage = _processInfo.GetMemoryUsage(); lock (_memoryLocker) { if (now >= _refreshAfterMemory) { - // Don't change calculation order, otherwise we loose some precision: - _memoryPercentage = Math.Min(_metricValueMultiplier, memoryUsage / _memoryLimit * _metricValueMultiplier); + _memoryUsage = memoryUsage; _refreshAfterMemory = now.Add(_memoryRefreshInterval); + _logger.ContainerMemoryUsageData(_memoryUsage, _memoryLimit); } - _logger.MemoryUsageData(memoryUsage, _memoryLimit, _memoryPercentage); - - return _memoryPercentage; + return _memoryUsage; } } diff --git a/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/WindowsSnapshotProvider.cs b/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/WindowsSnapshotProvider.cs index 837cd0f9a06..e238a59aff9 100644 --- a/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/WindowsSnapshotProvider.cs +++ b/src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/WindowsSnapshotProvider.cs @@ -144,7 +144,7 @@ private double MemoryPercentage() _refreshAfterMemory = now.Add(_memoryRefreshInterval); } - _logger.MemoryUsageData((ulong)currentMemoryUsage, _totalMemory, _memoryPercentage); + _logger.ProcessMemoryPercentageData((ulong)currentMemoryUsage, _totalMemory, _memoryPercentage); return _memoryPercentage; } diff --git a/src/Shared/Instruments/ResourceUtilizationInstruments.cs b/src/Shared/Instruments/ResourceUtilizationInstruments.cs index 835d3099782..b1593edd396 100644 --- a/src/Shared/Instruments/ResourceUtilizationInstruments.cs +++ b/src/Shared/Instruments/ResourceUtilizationInstruments.cs @@ -50,6 +50,14 @@ internal static class ResourceUtilizationInstruments /// public const string ContainerMemoryLimitUtilization = "container.memory.limit.utilization"; + /// + /// The name of an instrument to retrieve memory usage measured in bytes of all processes running inside a container or control group. + /// + /// + /// The type of an instrument is . + /// + public const string ContainerMemoryUsage = "container.memory.usage"; + /// /// The name of an instrument to retrieve CPU consumption share of the running process in range [0, 1]. /// diff --git a/test/Libraries/Microsoft.Extensions.Diagnostics.HealthChecks.ResourceUtilization.Tests/ResourceHealthCheckExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.Diagnostics.HealthChecks.ResourceUtilization.Tests/ResourceHealthCheckExtensionsTests.cs index 7d9b347a59d..2599be2dd9e 100644 --- a/test/Libraries/Microsoft.Extensions.Diagnostics.HealthChecks.ResourceUtilization.Tests/ResourceHealthCheckExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.Diagnostics.HealthChecks.ResourceUtilization.Tests/ResourceHealthCheckExtensionsTests.cs @@ -490,6 +490,7 @@ public async Task TestCpuAndMemoryChecks_WithMetrics( Mock processInfoMock = new(); var appMemoryUsage = memoryUsed; processInfoMock.Setup(p => p.GetMemoryUsage()).Returns(() => appMemoryUsage); + processInfoMock.Setup(p => p.GetCurrentProcessMemoryUsage()).Returns(() => appMemoryUsage); JOBOBJECT_EXTENDED_LIMIT_INFORMATION limitInfo = default; limitInfo.JobMemoryLimit = new UIntPtr(totalMemory); diff --git a/test/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests/Linux/AcceptanceTest.cs b/test/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests/Linux/AcceptanceTest.cs index 0221efc27c9..a4107faef03 100644 --- a/test/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests/Linux/AcceptanceTest.cs +++ b/test/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests/Linux/AcceptanceTest.cs @@ -220,10 +220,20 @@ public Task ResourceUtilizationTracker_And_Metrics_Report_Same_Values_With_Cgrou using var e = new ManualResetEventSlim(); object? meterScope = null; - listener.InstrumentPublished = (Instrument instrument, MeterListener meterListener) - => OnInstrumentPublished(instrument, meterListener, meterScope); - listener.SetMeasurementEventCallback((m, f, tags, _) - => OnMeasurementReceived(m, f, tags, ref cpuUserTime, ref cpuKernelTime, ref cpuFromGauge, ref cpuLimitFromGauge, ref cpuRequestFromGauge, ref memoryFromGauge, ref memoryLimitFromGauge)); + listener.InstrumentPublished = (Instrument instrument, MeterListener meterListener) => + OnInstrumentPublished(instrument, meterListener, meterScope); + listener.SetMeasurementEventCallback((m, f, tags, _) => + OnMeasurementReceived( + m, + f, + tags, + ref cpuUserTime, + ref cpuKernelTime, + ref cpuFromGauge, + ref cpuLimitFromGauge, + ref cpuRequestFromGauge, + ref memoryFromGauge, + ref memoryLimitFromGauge)); listener.Start(); using var host = FakeHost.CreateBuilder() @@ -302,13 +312,31 @@ public Task ResourceUtilizationTracker_And_Metrics_Report_Same_Values_With_Cgrou var cpuRequestFromGauge = 0.0d; var memoryFromGauge = 0.0d; var memoryLimitFromGauge = 0.0d; + long memoryUsageFromGauge = 0; using var e = new ManualResetEventSlim(); object? meterScope = null; - listener.InstrumentPublished = (Instrument instrument, MeterListener meterListener) - => OnInstrumentPublished(instrument, meterListener, meterScope); - listener.SetMeasurementEventCallback((m, f, tags, _) - => OnMeasurementReceived(m, f, tags, ref cpuUserTime, ref cpuKernelTime, ref cpuFromGauge, ref cpuLimitFromGauge, ref cpuRequestFromGauge, ref memoryFromGauge, ref memoryLimitFromGauge)); + listener.InstrumentPublished = (Instrument instrument, MeterListener meterListener) => + OnInstrumentPublished(instrument, meterListener, meterScope); + listener.SetMeasurementEventCallback((m, f, tags, _) => + OnMeasurementReceived( + m, + f, + tags, + ref cpuUserTime, + ref cpuKernelTime, + ref cpuFromGauge, + ref cpuLimitFromGauge, + ref cpuRequestFromGauge, + ref memoryFromGauge, + ref memoryLimitFromGauge)); + listener.SetMeasurementEventCallback((instrument, value, tags, _) => + { + if (instrument.Name == ResourceUtilizationInstruments.ContainerMemoryUsage) + { + memoryUsageFromGauge = value; + } + }); listener.Start(); using var host = FakeHost.CreateBuilder() @@ -355,6 +383,7 @@ public Task ResourceUtilizationTracker_And_Metrics_Report_Same_Values_With_Cgrou Assert.Equal(1, roundedCpuUsedPercentage); Assert.Equal(50, utilization.MemoryUsedPercentage); + Assert.Equal(524288, memoryUsageFromGauge); Assert.Equal(0.5, cpuLimitFromGauge * 100); Assert.Equal(roundedCpuUsedPercentage, Math.Round(cpuRequestFromGauge * 100)); Assert.Equal(utilization.MemoryUsedPercentage, memoryLimitFromGauge * 100); @@ -392,10 +421,11 @@ public Task ResourceUtilizationTracker_And_Metrics_Report_Same_Values_With_Cgrou var memoryLimitFromGauge = 0.0d; object? meterScope = null; - listener.InstrumentPublished = (Instrument instrument, MeterListener meterListener) - => OnInstrumentPublished(instrument, meterListener, meterScope); - listener.SetMeasurementEventCallback((m, f, tags, _) - => OnMeasurementReceived(m, + listener.InstrumentPublished = (Instrument instrument, MeterListener meterListener) => + OnInstrumentPublished(instrument, meterListener, meterScope); + listener.SetMeasurementEventCallback((m, f, tags, _) => + OnMeasurementReceived( + m, f, tags, ref cpuUserTime, @@ -455,7 +485,8 @@ private static void OnInstrumentPublished(Instrument instrument, MeterListener m instrument.Name == ResourceUtilizationInstruments.ContainerCpuTime || instrument.Name == ResourceUtilizationInstruments.ContainerCpuRequestUtilization || instrument.Name == ResourceUtilizationInstruments.ContainerCpuLimitUtilization || - instrument.Name == ResourceUtilizationInstruments.ContainerMemoryLimitUtilization) + instrument.Name == ResourceUtilizationInstruments.ContainerMemoryLimitUtilization || + instrument.Name == ResourceUtilizationInstruments.ContainerMemoryUsage) { meterListener.EnableMeasurementEvents(instrument); } diff --git a/test/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests/Linux/LinuxUtilizationProviderTests.cs b/test/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests/Linux/LinuxUtilizationProviderTests.cs index b34dfb1c258..c60ec5fa834 100644 --- a/test/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests/Linux/LinuxUtilizationProviderTests.cs +++ b/test/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests/Linux/LinuxUtilizationProviderTests.cs @@ -72,10 +72,18 @@ public void Provider_Registers_Instruments() } }); + listener.SetMeasurementEventCallback((instrument, value, _, _) => + { + if (ReferenceEquals(meter, instrument.Meter)) + { + samples.Add((instrument, value)); + } + }); + listener.Start(); listener.RecordObservableInstruments(); - Assert.Equal(5, samples.Count); + Assert.Equal(6, samples.Count); Assert.Contains(samples, x => x.instrument.Name == ResourceUtilizationInstruments.ContainerCpuLimitUtilization); Assert.True(double.IsNaN(samples.Single(i => i.instrument.Name == ResourceUtilizationInstruments.ContainerCpuLimitUtilization).value)); @@ -86,6 +94,9 @@ public void Provider_Registers_Instruments() Assert.Contains(samples, x => x.instrument.Name == ResourceUtilizationInstruments.ContainerMemoryLimitUtilization); Assert.Equal(0.5, samples.Single(i => i.instrument.Name == ResourceUtilizationInstruments.ContainerMemoryLimitUtilization).value); + Assert.Contains(samples, x => x.instrument.Name == ResourceUtilizationInstruments.ContainerMemoryUsage); + Assert.Equal(524288, samples.Single(i => i.instrument.Name == ResourceUtilizationInstruments.ContainerMemoryUsage).value); + Assert.Contains(samples, x => x.instrument.Name == ResourceUtilizationInstruments.ProcessCpuUtilization); Assert.True(double.IsNaN(samples.Single(i => i.instrument.Name == ResourceUtilizationInstruments.ProcessCpuUtilization).value)); @@ -359,7 +370,7 @@ public void Provider_GetMeasurementWithRetry_UnhandledException_DoesNotBlockFutu parserMock.Setup(p => p.GetMemoryUsageInBytes()).Returns(() => { callCount++; - if (callCount <= 2) + if (callCount <= 3) { throw new InvalidOperationException("Simulated unhandled exception"); } @@ -403,6 +414,6 @@ public void Provider_GetMeasurementWithRetry_UnhandledException_DoesNotBlockFutu var metric = samples.SingleOrDefault(x => x.instrument.Name == ResourceUtilizationInstruments.ProcessMemoryUtilization); Assert.Equal(1234f / 2000f, metric.value, 0.01f); - parserMock.Verify(p => p.GetMemoryUsageInBytes(), Times.Exactly(3)); + parserMock.Verify(p => p.GetMemoryUsageInBytes(), Times.Exactly(4)); } } diff --git a/test/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests/Windows/WindowsContainerSnapshotProviderTests.cs b/test/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests/Windows/WindowsContainerSnapshotProviderTests.cs index fc9113eacc9..ead512e015b 100644 --- a/test/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests/Windows/WindowsContainerSnapshotProviderTests.cs +++ b/test/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests/Windows/WindowsContainerSnapshotProviderTests.cs @@ -379,6 +379,146 @@ public void SnapshotProvider_EmitsMemoryMetrics(string instrumentName, bool useZ Assert.Equal(0.3 * multiplier, metricCollector.LastMeasurement.Value); // Consuming 30% of the memory afterwards. } + [Fact] + public void SnapshotProvider_TestMemoryMetricsTogether() + { + _appMemoryUsage = 200UL; + ulong containerMemoryUsage = 400UL; + ulong updatedAppMemoryUsage = 600UL; + ulong updatedContainerMemoryUsage = 1200UL; + + _processInfoMock.SetupSequence(p => p.GetCurrentProcessMemoryUsage()) + .Returns(() => _appMemoryUsage) + .Returns(updatedAppMemoryUsage) + .Throws(new InvalidOperationException("We shouldn't hit here...")); + + _processInfoMock.SetupSequence(p => p.GetMemoryUsage()) + .Returns(() => containerMemoryUsage) + .Returns(updatedContainerMemoryUsage) + .Throws(new InvalidOperationException("We shouldn't hit here...")); + + var fakeClock = new FakeTimeProvider(); + using var meter = new Meter(nameof(SnapshotProvider_TestMemoryMetricsTogether)); + var meterFactoryMock = new Mock(); + meterFactoryMock.Setup(x => x.Create(It.IsAny())) + .Returns(meter); + using var processMetricCollector = new MetricCollector(meter, ResourceUtilizationInstruments.ProcessMemoryUtilization, fakeClock); + using var containerLimitMetricCollector = new MetricCollector(meter, ResourceUtilizationInstruments.ContainerMemoryLimitUtilization, fakeClock); + using var containerUsageMetricCollector = new MetricCollector(meter, ResourceUtilizationInstruments.ContainerMemoryUsage, fakeClock); + + var options = new ResourceMonitoringOptions + { + MemoryConsumptionRefreshInterval = TimeSpan.FromMilliseconds(2) + }; + var snapshotProvider = new WindowsContainerSnapshotProvider( + _memoryInfoMock.Object, + _systemInfoMock.Object, + _processInfoMock.Object, + _logger, + meterFactoryMock.Object, + () => _jobHandleMock.Object, + fakeClock, + options); + + // Step #0 - state in the beginning: + processMetricCollector.RecordObservableInstruments(); + containerLimitMetricCollector.RecordObservableInstruments(); + containerUsageMetricCollector.RecordObservableInstruments(); + + Assert.NotNull(processMetricCollector.LastMeasurement?.Value); + Assert.NotNull(containerLimitMetricCollector.LastMeasurement?.Value); + Assert.NotNull(containerUsageMetricCollector.LastMeasurement?.Value); + + Assert.Equal(10, processMetricCollector.LastMeasurement.Value); // Process is consuming 10% of memory limit initially. + Assert.Equal(20, containerLimitMetricCollector.LastMeasurement.Value); // The whole container is consuming 20% of the memory limit initially. + Assert.Equal((long)containerMemoryUsage, containerUsageMetricCollector.LastMeasurement.Value); // 400 bytes of memory usage initially. + + // Step #1 - simulate 1 millisecond passing and collect metrics again: + fakeClock.Advance(options.MemoryConsumptionRefreshInterval - TimeSpan.FromMilliseconds(1)); + + processMetricCollector.RecordObservableInstruments(); + containerUsageMetricCollector.RecordObservableInstruments(); + containerLimitMetricCollector.RecordObservableInstruments(); + + // Still consuming 10% and 20% as values weren't updated yet - not enough time passed. + Assert.Equal(10, processMetricCollector.LastMeasurement.Value); + Assert.Equal(20, containerLimitMetricCollector.LastMeasurement.Value); + Assert.Equal((long)containerMemoryUsage, containerUsageMetricCollector.LastMeasurement.Value); + + // Step #2 - simulate 2 milliseconds passing and collect metrics again: + fakeClock.Advance(TimeSpan.FromMilliseconds(1)); + + processMetricCollector.RecordObservableInstruments(); + containerLimitMetricCollector.RecordObservableInstruments(); + containerUsageMetricCollector.RecordObservableInstruments(); + + // App is consuming 30%, and container is consuming 60% of the limit: + Assert.Equal(30, processMetricCollector.LastMeasurement.Value); + Assert.Equal(60, containerLimitMetricCollector.LastMeasurement.Value); + Assert.Equal((long)updatedContainerMemoryUsage, containerUsageMetricCollector.LastMeasurement.Value); + } + + [Fact] + public void SnapshotProvider_EmitsMemoryUsageMetric() + { + _appMemoryUsage = 200UL; + const ulong UpdatedAppMemoryUsage = 600UL; + const ulong UpdatedAppMemoryUsage2 = 300UL; + + _processInfoMock.SetupSequence(p => p.GetCurrentProcessMemoryUsage()) + .Returns(() => _appMemoryUsage) + .Returns(UpdatedAppMemoryUsage) + .Returns(UpdatedAppMemoryUsage2) + .Throws(new InvalidOperationException("We shouldn't hit here...")); + + _processInfoMock.SetupSequence(p => p.GetMemoryUsage()) + .Returns(() => _appMemoryUsage) + .Returns(UpdatedAppMemoryUsage) + .Returns(UpdatedAppMemoryUsage2) + .Throws(new InvalidOperationException("We shouldn't hit here...")); + + var fakeClock = new FakeTimeProvider(); + using var meter = new Meter(nameof(SnapshotProvider_EmitsMemoryMetrics)); + var meterFactoryMock = new Mock(); + meterFactoryMock.Setup(x => x.Create(It.IsAny())) + .Returns(meter); + using var metricCollector = new MetricCollector(meter, ResourceUtilizationInstruments.ContainerMemoryUsage, fakeClock); + + var options = new ResourceMonitoringOptions + { + MemoryConsumptionRefreshInterval = TimeSpan.FromMilliseconds(2), + }; + var snapshotProvider = new WindowsContainerSnapshotProvider( + _memoryInfoMock.Object, + _systemInfoMock.Object, + _processInfoMock.Object, + _logger, + meterFactoryMock.Object, + () => _jobHandleMock.Object, + fakeClock, + options); + + // Step #0 - state in the beginning: + metricCollector.RecordObservableInstruments(); + Assert.NotNull(metricCollector.LastMeasurement?.Value); + Assert.Equal(200, metricCollector.LastMeasurement.Value); // Consuming 200 bytes initially. + + // Step #1 - simulate 1 millisecond passing and collect metrics again: + fakeClock.Advance(options.MemoryConsumptionRefreshInterval - TimeSpan.FromMilliseconds(1)); + metricCollector.RecordObservableInstruments(); + Assert.Equal(200, metricCollector.LastMeasurement.Value); // Still consuming 200 bytes as metric wasn't updated. + + // Step #2 - simulate 2 milliseconds passing and collect metrics again: + fakeClock.Advance(TimeSpan.FromMilliseconds(2)); + metricCollector.RecordObservableInstruments(); + Assert.Equal(600, metricCollector.LastMeasurement.Value); // Consuming 600 bytes. + + // Step #3 - simulate 2 milliseconds passing and collect metrics again: + fakeClock.Advance(TimeSpan.FromMilliseconds(2)); + metricCollector.RecordObservableInstruments(); + Assert.Equal(300, metricCollector.LastMeasurement.Value); // Consuming 300 bytes. + } + [Fact] public Task SnapshotProvider_EmitsLogRecord() {