-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ci: added nightly-running chaos test supported by NBomber stress-test
- Loading branch information
Showing
12 changed files
with
335 additions
and
45 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,5 +2,7 @@ | |
| !src | ||
| !.editorconfig | ||
| !*.sln | ||
| !Directory.Build.props | ||
| !tests/chaos | ||
| src/**/bin | ||
| src/**/obj | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| name: nightly chaos testing | ||
|
|
||
| on: | ||
| schedule: | ||
| # daily at 04:07. | ||
| - cron: "07 04 * * *" | ||
|
|
||
| # Declare default permissions as read only. | ||
| permissions: read-all | ||
|
|
||
| jobs: | ||
| chaos-testing: | ||
| name: chaos testing | ||
| runs-on: ubuntu-22.04 | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3 | ||
| with: | ||
| fetch-depth: 0 | ||
|
|
||
| - name: Build stress testing image | ||
| id: build-image | ||
| uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # tag=v3 | ||
| with: | ||
| push: false | ||
| load: true | ||
| tags: ghcr.io/miracum/vfps/stress-test:v1 | ||
| cache-from: type=gha | ||
| cache-to: type=gha,mode=max | ||
| target: stress-test | ||
|
|
||
| - name: Create KinD cluster | ||
| uses: helm/kind-action@9e8295d178de23cbfbd8fa16cf844eec1d773a07 # tag=v1.4.0 | ||
| with: | ||
| cluster_name: kind | ||
|
|
||
| - name: Load stress-test image into KinD | ||
| run: | | ||
| kind load docker-image ghcr.io/miracum/vfps/stress-test:v1 | ||
| - name: Install prerequisites | ||
| working-directory: tests/chaos | ||
| run: | | ||
| curl -sL -o - https://github.com/argoproj/argo-workflows/releases/download/v3.4.3/argo-linux-amd64.gz | gunzip > argo | ||
| kubectl create ns vfps | ||
| helm repo add chaos-mesh https://charts.chaos-mesh.org | ||
| helm upgrade --install chaos-mesh chaos-mesh/chaos-mesh \ | ||
| --create-namespace \ | ||
| --wait \ | ||
| -n chaos-mesh \ | ||
| --set chaosDaemon.runtime=containerd \ | ||
| --set chaosDaemon.socketPath='/run/containerd/containerd.sock' \ | ||
| --version 2.4.3 | ||
| kubectl apply -f chaos-mesh-rbac.yaml | ||
| helm repo add argo https://argoproj.github.io/argo-helm | ||
| helm upgrade --install argo-workflows argo/argo-workflows \ | ||
| --create-namespace \ | ||
| --wait \ | ||
| -n argo-workflows \ | ||
| -f argo-workflows-values.yaml | ||
| - name: Install vfps | ||
| working-directory: tests/chaos | ||
| run: | | ||
| helm repo add miracum https://miracum.github.io/charts | ||
| helm upgrade --install \ | ||
| -n vfps \ | ||
| -f vfps-values.yaml \ | ||
| --wait \ | ||
| --version=^1.0.0 \ | ||
| vfps miracum/vfps | ||
| - name: Run chaos testing workflow | ||
| working-directory: tests/chaos | ||
| run: | | ||
| ./argo submit tests/chaos/argo-workflow.yaml -n vfps --wait |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -405,3 +405,6 @@ coverage/ | |
| coveragereport/ | ||
|
|
||
| megalinter-reports/ | ||
|
|
||
| # used by NBomber | ||
| reports/ | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,54 +1,96 @@ | ||
| using System.Diagnostics; | ||
| using Grpc.Core; | ||
| using Grpc.Net.Client; | ||
| using Grpc.Net.Client.Configuration; | ||
| using NBomber.Contracts; | ||
| using NBomber.CSharp; | ||
| using NBomber.Plugins.Http.CSharp; | ||
| using NBomber.Plugins.Network.Ping; | ||
| using System.Net.Http.Json; | ||
| using Vfps.Protos; | ||
|
|
||
| var baseUrl = "https://localhost:7078/v1"; | ||
| var grpcAddress = new Uri(Environment.GetEnvironmentVariable("VFPS_GRPC_ADDRESS") ?? "http://localhost:8081"); | ||
|
|
||
| var httpFactory = HttpClientFactory.Create(); | ||
| var defaultMethodConfig = new MethodConfig | ||
| { | ||
| Names = { MethodName.Default }, | ||
| RetryPolicy = new RetryPolicy | ||
| { | ||
| MaxAttempts = 3, | ||
| InitialBackoff = TimeSpan.FromSeconds(5), | ||
| MaxBackoff = TimeSpan.FromSeconds(30), | ||
| BackoffMultiplier = 2, | ||
| RetryableStatusCodes = { StatusCode.Unavailable, StatusCode.Internal } | ||
| } | ||
| }; | ||
|
|
||
| var createNamespace = Step.Create("create_namespace", | ||
| clientFactory: httpFactory, | ||
| execute: context => | ||
| { | ||
| var request = Http.CreateRequest("POST", baseUrl + "/namespaces") | ||
| .WithHeader("Accept", "application/json") | ||
| .WithBody(JsonContent.Create(new | ||
| { | ||
| Name = "load-test", | ||
| PseudonymLength = 32, | ||
| PseudonymGenerationMethod = 1, | ||
| })); | ||
|
|
||
| return Http.Send(request, context); | ||
| }); | ||
| using var channel = GrpcChannel.ForAddress(grpcAddress, new GrpcChannelOptions() | ||
| { | ||
| ServiceConfig = new ServiceConfig() | ||
| { | ||
| MethodConfigs = { defaultMethodConfig } | ||
| }, | ||
| Credentials = ChannelCredentials.Insecure, | ||
| UnsafeUseInsecureChannelCallCredentials = true, | ||
| }); | ||
|
|
||
| var namespaceClient = new NamespaceService.NamespaceServiceClient(channel); | ||
| var pseudonymClient = new PseudonymService.PseudonymServiceClient(channel); | ||
|
|
||
| var namespaceRequest = new NamespaceServiceCreateRequest() | ||
| { | ||
| Name = "stress-test", | ||
| PseudonymGenerationMethod = PseudonymGenerationMethod.SecureRandomBase64UrlEncoded, | ||
| PseudonymLength = 16, | ||
| PseudonymPrefix = "stress-", | ||
| }; | ||
|
|
||
| var createPseudonyms = Step.Create("create_pseudonyms", | ||
| clientFactory: httpFactory, | ||
| execute: context => | ||
| execute: async context => | ||
| { | ||
| var request = Http.CreateRequest("POST", baseUrl + "/namespaces/load-test/pseudonyms") | ||
| .WithHeader("Accept", "application/json") | ||
| .WithBody(JsonContent.Create(new | ||
| { | ||
| OriginalValue = Guid.NewGuid().ToString(), | ||
| })); | ||
|
|
||
| return Http.Send(request, context); | ||
| var request = new PseudonymServiceCreateRequest() | ||
| { | ||
| Namespace = namespaceRequest.Name, | ||
| OriginalValue = Guid.NewGuid().ToString(), | ||
| }; | ||
|
|
||
| try | ||
| { | ||
| var response = await pseudonymClient.CreateAsync(request); | ||
| return Response.Ok(statusCode: 200, sizeBytes: request.CalculateSize() + response.CalculateSize()); | ||
| } | ||
| catch (RpcException exc) | ||
| { | ||
| context.Logger.Error(exc, "Pseudonym creation failed"); | ||
| return Response.Fail(); | ||
| } | ||
| }); | ||
|
|
||
| var scenario = ScenarioBuilder | ||
| .CreateScenario("create_namespace_and_pseudonyms", createNamespace, createPseudonyms) | ||
| .CreateScenario("stress_pseudonym_creation", createPseudonyms) | ||
| .WithInit(async context => | ||
| { | ||
| try | ||
| { | ||
| var response = await namespaceClient.CreateAsync(namespaceRequest); | ||
| } | ||
| catch (RpcException exc) when (exc.StatusCode == StatusCode.AlreadyExists) | ||
| { | ||
| context.Logger.Warning($"Namespace {namespaceRequest.Name} already exists. Continuing anyway."); | ||
| } | ||
| }) | ||
| .WithWarmUpDuration(TimeSpan.FromSeconds(5)) | ||
| .WithLoadSimulations( | ||
| Simulation.InjectPerSec(rate: 100, during: TimeSpan.FromSeconds(30)) | ||
| Simulation.RampConstant(copies: 10, during: TimeSpan.FromMinutes(4)), | ||
| Simulation.KeepConstant(copies: 100, during: TimeSpan.FromMinutes(4)), | ||
| Simulation.InjectPerSecRandom(minRate: 10, maxRate: 50, during: TimeSpan.FromMinutes(4)) | ||
| ); | ||
|
|
||
| // creates ping plugin that brings additional reporting data | ||
| var pingPluginConfig = PingPluginConfig.CreateDefault(new[] { "127.0.0.1" }); | ||
| var pingPluginConfig = PingPluginConfig.CreateDefault(new[] { grpcAddress.Host }); | ||
| var pingPlugin = new PingPlugin(pingPluginConfig); | ||
|
|
||
| NBomberRunner | ||
| var stats = NBomberRunner | ||
| .RegisterScenarios(scenario) | ||
| .WithWorkerPlugins(pingPlugin) | ||
| .Run(); | ||
|
|
||
| Debug.Assert(stats.FailCount < 100); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.3/api/jsonschema/schema.json | ||
| apiVersion: argoproj.io/v1alpha1 | ||
| kind: Workflow | ||
| metadata: | ||
| generateName: vfps-chaos-workflow- | ||
| spec: | ||
| entrypoint: run-chaos-and-test | ||
| templates: | ||
| - name: test | ||
| serviceAccountName: chaos-mesh-cluster-manager | ||
| container: | ||
| image: ghcr.io/miracum/vfps/stress-test:v1 | ||
| imagePullPolicy: IfNotPresent | ||
| command: | ||
| - dotnet | ||
| args: | ||
| - /opt/vfps-stress/Vfps.LoadTests.dll | ||
| env: | ||
| - name: VFPS_GRPC_ADDRESS | ||
| value: dns:///vfps-headless:8081 | ||
| securityContext: | ||
| allowPrivilegeEscalation: false | ||
| capabilities: | ||
| drop: | ||
| - ALL | ||
| privileged: false | ||
| runAsNonRoot: true | ||
|
|
||
| - name: install-chaos | ||
| serviceAccountName: chaos-mesh-cluster-manager | ||
| container: | ||
| image: ghcr.io/miracum/vfps/stress-test:v1 | ||
| imagePullPolicy: IfNotPresent | ||
| command: | ||
| - kubectl | ||
| args: | ||
| - apply | ||
| - -f | ||
| - /tmp/chaos.yaml | ||
| securityContext: | ||
| allowPrivilegeEscalation: false | ||
| capabilities: | ||
| drop: | ||
| - ALL | ||
| privileged: false | ||
| runAsNonRoot: true | ||
|
|
||
| - name: delete-chaos | ||
| serviceAccountName: chaos-mesh-cluster-manager | ||
| container: | ||
| image: ghcr.io/miracum/vfps/stress-test:v1 | ||
| imagePullPolicy: IfNotPresent | ||
| command: | ||
| - kubectl | ||
| args: | ||
| - delete | ||
| - -f | ||
| - /tmp/chaos.yaml | ||
| securityContext: | ||
| allowPrivilegeEscalation: false | ||
| capabilities: | ||
| drop: | ||
| - ALL | ||
| privileged: false | ||
| runAsNonRoot: true | ||
|
|
||
| - name: run-chaos-and-test | ||
| dag: | ||
| tasks: | ||
| - name: test | ||
| template: test | ||
| - name: install-chaos | ||
| template: install-chaos | ||
| - name: delete-chaos | ||
| depends: "install-chaos && (test.Succeeded || test.Failed)" | ||
| template: delete-chaos |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| controller: | ||
| workflowNamespaces: | ||
| - default | ||
| - argo-workflows | ||
| - vfps | ||
|
|
||
| server: | ||
| extraArgs: [--auth-mode=server] |
Oops, something went wrong.