Skip to content
This repository has been archived by the owner on Nov 1, 2023. It is now read-only.

Commit

Permalink
Port SyncAutoscaleSettings from Python to C#
Browse files Browse the repository at this point in the history
  • Loading branch information
stas committed Sep 15, 2022
1 parent f22dee1 commit cbd724c
Show file tree
Hide file tree
Showing 4 changed files with 126 additions and 10 deletions.
6 changes: 3 additions & 3 deletions src/ApiService/ApiService/Functions/TimerTasks.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,20 +34,20 @@ public async Async.Task Run([TimerTrigger("00:00:15")] TimerInfo myTimer) {

await foreach (var job in expiredJobs) {
_logger.Info($"stopping expired job. job_id:{job.JobId}");
await _jobOperations.Stopping(job);
var _ = await _jobOperations.Stopping(job);
}

var jobs = _jobOperations.SearchState(states: JobStateHelper.NeedsWork);

await foreach (var job in jobs) {
_logger.Info($"update job: {job.JobId}");
await _jobOperations.ProcessStateUpdates(job);
var _ = await _jobOperations.ProcessStateUpdates(job);
}

var tasks = _taskOperations.SearchStates(states: TaskStateHelper.NeedsWorkStates);
await foreach (var task in tasks) {
_logger.Info($"update task: {task.TaskId}");
await _taskOperations.ProcessStateUpdate(task);
var _ = await _taskOperations.ProcessStateUpdate(task);
}

await _scheduler.ScheduleTasks();
Expand Down
7 changes: 4 additions & 3 deletions src/ApiService/ApiService/Functions/TimerWorkers.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ public async Async.Task ProcessScalesets(Service.Scaleset scaleset) {
_log.Verbose($"checking scaleset for updates: {scaleset.ScalesetId}");

await _scaleSetOps.UpdateConfigs(scaleset);
await _scaleSetOps.SyncAutoscaleSettings(scaleset);

// if the scaleset is touched during cleanup, don't continue to process it
if (await _scaleSetOps.CleanupNodes(scaleset)) {
Expand All @@ -27,7 +28,7 @@ public async Async.Task ProcessScalesets(Service.Scaleset scaleset) {
}

await _scaleSetOps.SyncScalesetSize(scaleset);
await _scaleSetOps.ProcessStateUpdate(scaleset);
var _ = await _scaleSetOps.ProcessStateUpdate(scaleset);
}


Expand All @@ -41,7 +42,7 @@ public async Async.Task Run([TimerTrigger("00:01:30")] TimerInfo t) {
await foreach (var pool in pools) {
if (PoolStateHelper.NeedsWork.Contains(pool.State)) {
_log.Info($"update pool: {pool.PoolId} ({pool.Name})");
await _poolOps.ProcessStateUpdate(pool);
var _ = await _poolOps.ProcessStateUpdate(pool);
}
}

Expand All @@ -57,7 +58,7 @@ public async Async.Task Run([TimerTrigger("00:01:30")] TimerInfo t) {
var nodes = _nodeOps.SearchStates(states: NodeStateHelper.NeedsWorkStates);
await foreach (var node in nodes) {
_log.Info($"update node: {node.MachineId}");
await _nodeOps.ProcessStateUpdate(node);
var _ = await _nodeOps.ProcessStateUpdate(node);
}

var scalesets = _scaleSetOps.SearchAll();
Expand Down
62 changes: 59 additions & 3 deletions src/ApiService/ApiService/onefuzzlib/AutoScale.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,18 @@ AutoscaleProfile CreateAutoScaleProfile(
OneFuzzResult<AutoscaleSettingResource?> GetAutoscaleSettings(Guid vmss);

Async.Task<OneFuzzResultVoid> UpdateAutoscale(AutoscaleSettingData autoscale);

Async.Task<OneFuzzResult<AutoscaleProfile>> GetAutoScaleProfile(Guid scalesetId);

Async.Task<AutoScale> Update(
Guid scalesetId,
long minAmount,
long maxAmount,
long defaultAmount,
long scaleOutAmount,
long scaleOutCooldown,
long scaleInAmount,
long scaleInCooldown);
}


Expand Down Expand Up @@ -77,6 +89,25 @@ public async Async.Task<AutoScale> Create(
}
}

public async Async.Task<OneFuzzResult<AutoscaleProfile>> GetAutoScaleProfile(Guid scalesetId) {
_logTracer.Info($"getting scaleset for existing auto-scale resources {scalesetId}");
var settings = _context.Creds.GetResourceGroupResource().GetAutoscaleSettings();
if (settings is null) {
return OneFuzzResult<AutoscaleProfile>.Error(ErrorCode.INVALID_CONFIGURATION, $"could not find any auto-scale settings for the resource group");
} else {
await foreach (var setting in settings.GetAllAsync()) {

if (setting.Data.TargetResourceId.EndsWith(scalesetId.ToString())) {
if (setting.Data.Profiles.Count != 1) {
return OneFuzzResult<AutoscaleProfile>.Error(ErrorCode.INVALID_CONFIGURATION, $"found {setting.Data.Profiles.Count} auto-scale profiles for {scalesetId}");
} else {
return OneFuzzResult<AutoscaleProfile>.Ok(setting.Data.Profiles.First());
}
}
}
}
return OneFuzzResult<AutoscaleProfile>.Error(ErrorCode.INVALID_CONFIGURATION, $"could not find auto-scale settings for scaleset {scalesetId}");
}

public async Async.Task<OneFuzzResultVoid> AddAutoScaleToVmss(Guid vmss, AutoscaleProfile autoScaleProfile) {
_logTracer.Info($"Checking scaleset {vmss} for existing auto scale resource");
Expand Down Expand Up @@ -253,9 +284,6 @@ private async Async.Task<OneFuzzResult<DiagnosticSettingsResource>> SetupAutoSca

public OneFuzzResult<AutoscaleSettingResource?> GetAutoscaleSettings(Guid vmss) {
_logTracer.Info($"Checking scaleset {vmss} for existing auto scale resource");
var monitorManagementClient = _context.LogAnalytics.GetMonitorManagementClient();
var resourceGroup = _context.Creds.GetBaseResourceGroup();

try {
var autoscale = _context.Creds.GetResourceGroupResource().GetAutoscaleSettings()
.ToEnumerable()
Expand Down Expand Up @@ -319,4 +347,32 @@ public static ScaleRule ShutdownScalesetRule(string queueUri) {
) { Value = "1" }
);
}

public async Async.Task<AutoScale> Update(
Guid scalesetId,
long minAmount,
long maxAmount,
long defaultAmount,
long scaleOutAmount,
long scaleOutCooldown,
long scaleInAmount,
long scaleInCooldown) {

var entry = new AutoScale(
scalesetId,
Min: minAmount,
Max: maxAmount,
Default: defaultAmount,
ScaleOutAmount: scaleOutAmount,
ScaleOutCooldown: scaleOutCooldown,
ScaleInAmount: scaleInAmount,
ScaleInCooldown: scaleInCooldown
);

var r = await Replace(entry);
if (!r.IsOk) {
_logTracer.Error($"Failed to replace auto-scale record for scaleset ID: {scalesetId}, minAmount: {minAmount}, maxAmount: {maxAmount}, defaultAmount: {defaultAmount}, scaleOutAmount: {scaleOutAmount}, scaleOutCooldown: {scaleOutCooldown}, scaleInAmount: {scaleInAmount}, scaleInCooldown: {scaleInCooldown}");
}
return entry;
}
}
61 changes: 60 additions & 1 deletion src/ApiService/ApiService/onefuzzlib/ScalesetOperations.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
using ApiService.OneFuzzLib.Orm;
using Azure.ResourceManager.Compute;
using Azure.ResourceManager.Monitor.Models;

namespace Microsoft.OneFuzz.Service;

public interface IScalesetOperations : IStatefulOrm<Scaleset, ScalesetState> {
Expand Down Expand Up @@ -34,6 +33,8 @@ public interface IScalesetOperations : IStatefulOrm<Scaleset, ScalesetState> {
Async.Task<Scaleset> Shutdown(Scaleset scaleset);
Async.Task<Scaleset> Halt(Scaleset scaleset);
Async.Task<Scaleset> CreationFailed(Scaleset scaleset);

Async.Task<OneFuzzResultVoid> SyncAutoscaleSettings(Scaleset scaleset);
}

public class ScalesetOperations : StatefulOrm<Scaleset, ScalesetState, ScalesetOperations>, IScalesetOperations {
Expand Down Expand Up @@ -84,6 +85,64 @@ public async Async.Task SyncScalesetSize(Scaleset scaleset) {
}
}

public async Async.Task<OneFuzzResultVoid> SyncAutoscaleSettings(Scaleset scaleset) {
if (scaleset.State != ScalesetState.Running)
return OneFuzzResultVoid.Ok;

_log.Info($"syncing auto-scale settings for scaleset {scaleset.ScalesetId}");

var autoscaleProfile = await _context.AutoScaleOperations.GetAutoScaleProfile(scaleset.ScalesetId);
if (!autoscaleProfile.IsOk) {
return autoscaleProfile.ErrorV;
}
var profile = autoscaleProfile.OkV;

var minAmount = Int64.Parse(profile.Capacity.Minimum);
var maxAmount = Int64.Parse(profile.Capacity.Maximum);
var defaultAmount = Int64.Parse(profile.Capacity.Default);

var scaleOutAmount = 1;
var scaleOutCooldown = 10L;
var scaleInAmount = 1;
var scaleInCooldown = 15L;

foreach (var rule in profile.Rules) {
var scaleAction = rule.ScaleAction;

if (scaleAction.Direction == ScaleDirection.Increase) {
scaleOutAmount = Int32.Parse(scaleAction.Value);
_logTracer.Info($"Scaleout cooldown in seconds. Before: {scaleOutCooldown}");
scaleOutCooldown = (long)scaleAction.Cooldown.TotalMinutes;
_logTracer.Info($"Scaleout cooldown in seconds. After: {scaleOutCooldown}");
} else if (scaleAction.Direction == ScaleDirection.Decrease) {
scaleInAmount = Int32.Parse(scaleAction.Value);
_logTracer.Info($"Scalin cooldown in seconds. Before: {scaleInCooldown}");
scaleInCooldown = (long)scaleAction.Cooldown.TotalMinutes;
_logTracer.Info($"Scalein cooldown in seconds. After: {scaleInCooldown}");
} else {
continue;
}
}

var poolResult = await _context.PoolOperations.GetByName(scaleset.PoolName);
if (!poolResult.IsOk) {
return poolResult.ErrorV;
}

_logTracer.Info($"Updating auto-scale entry for scaleset: {scaleset.ScalesetId}");
var _ = await _context.AutoScaleOperations.Update(
scalesetId: scaleset.ScalesetId,
minAmount: minAmount,
maxAmount: maxAmount,
defaultAmount: defaultAmount,
scaleOutAmount: scaleOutAmount,
scaleOutCooldown: scaleOutCooldown,
scaleInAmount: scaleInAmount,
scaleInCooldown: scaleInCooldown);

return OneFuzzResultVoid.Ok;
}

public async Async.Task<Scaleset> Resize(Scaleset scaleset) {

if (scaleset.State != ScalesetState.Resize) {
Expand Down

0 comments on commit cbd724c

Please sign in to comment.