From 82e8ebbcb8f460f9f8c3f4515142194249369919 Mon Sep 17 00:00:00 2001 From: Fabian von Feilitzsch Date: Mon, 26 Jul 2021 14:56:17 -0400 Subject: [PATCH] Add ability to configure storage to scorecard (#5028) * Add initial implementation of storage configuration support * add exec and gather logic * remove config labels for storage and replace with the v1alpha3 storage spec to hold the mountPath, mountPath in the spec triggers storage to be added * Bump operator-framework/api dependency * Regenerate samples Co-authored-by: jmccormick2001 Signed-off-by: Fabian von Feilitzsch --- go.mod | 2 +- go.sum | 3 +- internal/cmd/operator-sdk/scorecard/cmd.go | 6 +- internal/scorecard/kubeclient.go | 11 +- internal/scorecard/scorecard.go | 39 +++- internal/scorecard/storage.go | 221 ++++++++++++++++++ internal/scorecard/testpod.go | 7 +- .../bundle/tests/scorecard/config.yaml | 21 ++ .../bundle/tests/scorecard/config.yaml | 21 ++ .../bundle/tests/scorecard/config.yaml | 21 ++ .../bundle/tests/scorecard/config.yaml | 21 ++ .../en/docs/cli/operator-sdk_scorecard.md | 1 + 12 files changed, 363 insertions(+), 11 deletions(-) create mode 100644 internal/scorecard/storage.go diff --git a/go.mod b/go.mod index 52bb571663c..fafd3f49965 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2 github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.13.0 - github.com/operator-framework/api v0.10.0 + github.com/operator-framework/api v0.10.2 github.com/operator-framework/java-operator-plugins v0.0.0-20210708174638-463fb91f3d5e github.com/operator-framework/operator-lib v0.5.0 github.com/operator-framework/operator-registry v1.17.4 diff --git a/go.sum b/go.sum index 8d548f0e72e..5e52c8fc55d 100644 --- a/go.sum +++ b/go.sum @@ -822,8 +822,9 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/operator-framework/api v0.7.1/go.mod h1:L7IvLd/ckxJEJg/t4oTTlnHKAJIP/p51AvEslW3wYdY= -github.com/operator-framework/api v0.10.0 h1:TaxbgbrV8D3wnKNyrImZ2zjQVVHMQRc7piWLDmlGoEE= github.com/operator-framework/api v0.10.0/go.mod h1:tV0BUNvly7szq28ZPBXhjp1Sqg5yHCOeX19ui9K4vjI= +github.com/operator-framework/api v0.10.2 h1:fo8Bhyx1v46NdJIz2rUzfzNUpe1KDNPtVpHVpuxZnk0= +github.com/operator-framework/api v0.10.2/go.mod h1:tV0BUNvly7szq28ZPBXhjp1Sqg5yHCOeX19ui9K4vjI= github.com/operator-framework/java-operator-plugins v0.0.0-20210708174638-463fb91f3d5e h1:LMsT59IJqaLn7kD6DnZFy0IouRufXyJHTT+mXQrl9Ps= github.com/operator-framework/java-operator-plugins v0.0.0-20210708174638-463fb91f3d5e/go.mod h1:sGKGELFkUeRqElcyvyPC89bC76YnCL7MPMa13P0AQcw= github.com/operator-framework/operator-lib v0.5.0 h1:Jmhz/WjcstEyBBM9IFUiHEgKg5bd43uF4ej/ZY2S0rM= diff --git a/internal/cmd/operator-sdk/scorecard/cmd.go b/internal/cmd/operator-sdk/scorecard/cmd.go index 541c2b20f36..98b18b93486 100644 --- a/internal/cmd/operator-sdk/scorecard/cmd.go +++ b/internal/cmd/operator-sdk/scorecard/cmd.go @@ -49,6 +49,7 @@ type scorecardCmd struct { list bool skipCleanup bool waitTime time.Duration + testOutput string } func NewCmd() *cobra.Command { @@ -85,6 +86,8 @@ If the argument holds an image tag, it must be present remotely.`, "Disable resource cleanup after tests are run") scorecardCmd.Flags().DurationVarP(&c.waitTime, "wait-time", "w", 30*time.Second, "seconds to wait for tests to complete. Example: 35s") + scorecardCmd.Flags().StringVarP(&c.testOutput, "test-output", "t", "test-output", + "Test output directory.") return scorecardCmd } @@ -196,11 +199,12 @@ func (c *scorecardCmd) run() (err error) { ServiceAccount: c.serviceAccount, Namespace: scorecard.GetKubeNamespace(c.kubeconfig, c.namespace), BundlePath: c.bundle, + TestOutput: c.testOutput, BundleMetadata: metadata, } // Only get the client if running tests. - if runner.Client, err = scorecard.GetKubeClient(c.kubeconfig); err != nil { + if runner.Client, runner.RESTConfig, err = scorecard.GetKubeClient(c.kubeconfig); err != nil { return fmt.Errorf("error getting kubernetes client: %w", err) } diff --git a/internal/scorecard/kubeclient.go b/internal/scorecard/kubeclient.go index db67b59c94a..4131b2c4139 100644 --- a/internal/scorecard/kubeclient.go +++ b/internal/scorecard/kubeclient.go @@ -19,6 +19,7 @@ import ( "github.com/operator-framework/operator-sdk/internal/util/k8sutil" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" cruntime "sigs.k8s.io/controller-runtime/pkg/client/config" ) @@ -30,24 +31,24 @@ import ( // - in-cluster connection for when the sdk is run within a cluster instead of // the command line // TODO(joelanford): migrate scorecard use `internal/operator.Configuration` -func GetKubeClient(kubeconfig string) (client kubernetes.Interface, err error) { +func GetKubeClient(kubeconfig string) (client kubernetes.Interface, config *rest.Config, err error) { if kubeconfig != "" { os.Setenv(k8sutil.KubeConfigEnvVar, kubeconfig) } - config, err := cruntime.GetConfig() + config, err = cruntime.GetConfig() if err != nil { - return client, err + return client, config, err } // create the clientset clientset, err := kubernetes.NewForConfig(config) if err != nil { - return client, err + return client, config, err } - return clientset, err + return clientset, config, err } // GetKubeNamespace returns the kubernetes namespace to use diff --git a/internal/scorecard/scorecard.go b/internal/scorecard/scorecard.go index 3379501f89c..1487e9c4f0a 100644 --- a/internal/scorecard/scorecard.go +++ b/internal/scorecard/scorecard.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" registryutil "github.com/operator-framework/operator-sdk/internal/registry" ) @@ -47,8 +48,10 @@ type PodTestRunner struct { Namespace string ServiceAccount string BundlePath string + TestOutput string BundleMetadata registryutil.Labels Client kubernetes.Interface + RESTConfig *rest.Config configMapName string } @@ -75,6 +78,7 @@ func (o Scorecard) Run(ctx context.Context) (testOutput v1alpha3.TestList, err e if len(tests) == 0 { continue } + tests = o.setTestDefaults(tests) output := make(chan v1alpha3.Test, len(tests)) if stage.Parallel { @@ -107,6 +111,15 @@ func (o Scorecard) Run(ctx context.Context) (testOutput v1alpha3.TestList, err e return testOutput, err } +func (o Scorecard) setTestDefaults(tests []v1alpha3.TestConfiguration) []v1alpha3.TestConfiguration { + for i := range tests { + if len(tests[i].Storage.Spec.MountPath.Path) == 0 { + tests[i].Storage.Spec.MountPath.Path = o.Config.Storage.Spec.MountPath.Path + } + } + return tests +} + func (o Scorecard) runStageParallel(ctx context.Context, tests []v1alpha3.TestConfiguration, results chan<- v1alpha3.Test) { var wg sync.WaitGroup for _, t := range tests { @@ -172,6 +185,7 @@ func (r *PodTestRunner) Initialize(ctx context.Context) error { if err != nil { return fmt.Errorf("error creating ConfigMap %w", err) } + return nil } @@ -187,6 +201,7 @@ func (r FakeTestRunner) Cleanup(ctx context.Context) error { // Cleanup deletes pods and configmap resources from this test run func (r PodTestRunner) Cleanup(ctx context.Context) (err error) { + err = r.deletePods(ctx, r.configMapName) if err != nil { return err @@ -195,13 +210,20 @@ func (r PodTestRunner) Cleanup(ctx context.Context) (err error) { if err != nil { return err } + return nil } // RunTest executes a single test func (r PodTestRunner) RunTest(ctx context.Context, test v1alpha3.TestConfiguration) (*v1alpha3.TestStatus, error) { + // Create a Pod to run the test podDef := getPodDefinition(r.configMapName, test, r) + + if test.Storage.Spec.MountPath.Path != "" { + addStorageToPod(podDef, test.Storage.Spec.MountPath.Path) + } + pod, err := r.Client.CoreV1().Pods(r.Namespace).Create(ctx, podDef, metav1.CreateOptions{}) if err != nil { return nil, err @@ -212,6 +234,14 @@ func (r PodTestRunner) RunTest(ctx context.Context, test v1alpha3.TestConfigurat return nil, err } + // gather test output if necessary + if test.Storage.Spec.MountPath.Path != "" { + err := gatherTestOutput(r, test.Labels["suite"], test.Labels["test"], pod.Name, test.Storage.Spec.MountPath.Path) + if err != nil { + return nil, err + } + } + return r.getTestStatus(ctx, pod), nil } @@ -239,9 +269,14 @@ func (r PodTestRunner) waitForTestToComplete(ctx context.Context, p *v1.Pod) (er if err != nil { return true, fmt.Errorf("error getting pod %s %w", p.Name, err) } - if tmp.Status.Phase == v1.PodSucceeded || tmp.Status.Phase == v1.PodFailed { - return true, nil + for _, s := range tmp.Status.ContainerStatuses { + if s.Name == "scorecard-test" { + if s.State.Terminated != nil { + return true, nil + } + } } + return false, nil }) diff --git a/internal/scorecard/storage.go b/internal/scorecard/storage.go new file mode 100644 index 00000000000..3ec2e537481 --- /dev/null +++ b/internal/scorecard/storage.go @@ -0,0 +1,221 @@ +// Copyright 2021 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorecard + +import ( + "archive/tar" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/kubectl/pkg/scheme" +) + +const ( + StorageSidecarContainer = "scorecard-gather" +) + +func (r PodTestRunner) execInPod(podName, mountPath, containerName string) (io.Reader, io.Reader, error) { + cmd := []string{ + "tar", + "cf", + "-", + mountPath, + } + + stdoutReader, outStream := io.Pipe() + stderrReader, errStream := io.Pipe() + const tty = false + req := r.Client.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(podName). + Namespace(r.Namespace).SubResource("exec").Param("container", containerName) + req.VersionedParams( + &v1.PodExecOptions{ + Command: cmd, + Stdin: false, + Stdout: true, + Stderr: true, + TTY: tty, + }, + scheme.ParameterCodec, + ) + + exec, err := remotecommand.NewSPDYExecutor(r.RESTConfig, "POST", req.URL()) + if err != nil { + return nil, nil, err + } + + go func() { + defer outStream.Close() + defer errStream.Close() + err = exec.Stream(remotecommand.StreamOptions{ + Stdin: nil, + Stdout: outStream, + Stderr: errStream, + }) + if err != nil { + log.Error(err) + } + }() + return stdoutReader, stderrReader, err +} + +func getStoragePrefix(file string) string { + return strings.TrimLeft(file, "/") +} + +func untarAll(reader io.Reader, destDir, prefix string) error { + tarReader := tar.NewReader(reader) + for { + header, err := tarReader.Next() + if err != nil { + if err != io.EOF { + return err + } + break + } + + if !strings.HasPrefix(header.Name, prefix) { + return fmt.Errorf("tar contents corrupted") + } + + mode := header.FileInfo().Mode() + destFileName := filepath.Join(destDir, header.Name[len(prefix):]) + + baseName := filepath.Dir(destFileName) + if err := os.MkdirAll(baseName, 0755); err != nil { + return err + } + if header.FileInfo().IsDir() { + if err := os.MkdirAll(destFileName, 0755); err != nil { + return err + } + continue + } + + if mode&os.ModeSymlink != 0 { + linkname := header.Linkname + + if err := os.Symlink(linkname, destFileName); err != nil { + return err + } + } else { + outFile, err := os.Create(destFileName) + if err != nil { + return err + } + defer outFile.Close() + if _, err := io.Copy(outFile, tarReader); err != nil { + return err + } + if err := outFile.Close(); err != nil { + return err + } + } + } + + return nil +} + +func addStorageToPod(podDef *v1.Pod, mountPath string) { + + // add the emptyDir volume for storage to the test Pod + newVolume := v1.Volume{} + newVolume.Name = "scorecard-storage" + newVolume.VolumeSource = v1.VolumeSource{} + + podDef.Spec.Volumes = append(podDef.Spec.Volumes, newVolume) + + // add the storage sidecar container + storageContainer := v1.Container{ + Name: StorageSidecarContainer, + Image: "busybox", + ImagePullPolicy: v1.PullIfNotPresent, + Args: []string{ + "/bin/sh", + "-c", + //"trap 'echo TERM;exit 0' TERM;tail -f /dev/null", + "sleep 1000", + }, + VolumeMounts: []v1.VolumeMount{ + { + MountPath: mountPath, + Name: "scorecard-storage", + ReadOnly: true, + }, + }, + } + + podDef.Spec.Containers = append(podDef.Spec.Containers, storageContainer) + + // add the storage emptyDir volume into the test container + + vMount := v1.VolumeMount{ + MountPath: mountPath, + Name: "scorecard-storage", + ReadOnly: false, + } + podDef.Spec.Containers[0].VolumeMounts = append(podDef.Spec.Containers[0].VolumeMounts, vMount) + +} + +func gatherTestOutput(r PodTestRunner, suiteName, testName, podName, mountPath string) error { + + //exec into sidecar container, run tar, get reader + containerName := StorageSidecarContainer + stdoutReader, stderrReader, err := r.execInPod(podName, mountPath, containerName) + if err != nil { + return err + } + + srcPath := r.TestOutput + prefix := getStoragePrefix(srcPath) + prefix = path.Clean(prefix) + destPath := getDestPath(r.TestOutput, suiteName, testName) + err = untarAll(stdoutReader, destPath, prefix) + if err != nil { + return err + } + stderr, err := ioutil.ReadAll(stderrReader) + if err != nil { + return err + } + if len(stderr) > 0 { + destFileName := filepath.Join(destPath, "tar_stderr") + err = ioutil.WriteFile(destFileName, stderr, 0644) + if err != nil { + return err + } + } + + return nil +} + +func getDestPath(baseDir, suiteName, testName string) (destPath string) { + destPath = baseDir + string(os.PathSeparator) + if suiteName != "" { + destPath = destPath + suiteName + string(os.PathSeparator) + } + destPath = destPath + testName + return destPath +} diff --git a/internal/scorecard/testpod.go b/internal/scorecard/testpod.go index 40d80537cc9..c905a5fe35d 100644 --- a/internal/scorecard/testpod.go +++ b/internal/scorecard/testpod.go @@ -39,6 +39,7 @@ const ( // getPodDefinition fills out a Pod definition based on // information from the test func getPodDefinition(configMapName string, test v1alpha3.TestConfiguration, r PodTestRunner) *v1.Pod { + return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("scorecard-test-%s", rand.String(4)), @@ -126,7 +127,11 @@ func getPodDefinition(configMapName string, test v1alpha3.TestConfiguration, r P // getPodLog fetches the test results which are found in the pod log func getPodLog(ctx context.Context, client kubernetes.Interface, pod *v1.Pod) ([]byte, error) { - req := client.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{}) + podLogOptions := v1.PodLogOptions{ + Container: "scorecard-test", + } + + req := client.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOptions) podLogs, err := req.Stream(ctx) if err != nil { return nil, err diff --git a/testdata/ansible/memcached-operator/bundle/tests/scorecard/config.yaml b/testdata/ansible/memcached-operator/bundle/tests/scorecard/config.yaml index 2ff360df3fa..8cba919ca99 100644 --- a/testdata/ansible/memcached-operator/bundle/tests/scorecard/config.yaml +++ b/testdata/ansible/memcached-operator/bundle/tests/scorecard/config.yaml @@ -12,6 +12,9 @@ stages: labels: suite: basic test: basic-check-spec-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-bundle-validation @@ -19,6 +22,9 @@ stages: labels: suite: olm test: olm-bundle-validation-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-crds-have-validation @@ -26,6 +32,9 @@ stages: labels: suite: olm test: olm-crds-have-validation-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-crds-have-resources @@ -33,6 +42,9 @@ stages: labels: suite: olm test: olm-crds-have-resources-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-spec-descriptors @@ -40,6 +52,9 @@ stages: labels: suite: olm test: olm-spec-descriptors-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-status-descriptors @@ -47,3 +62,9 @@ stages: labels: suite: olm test: olm-status-descriptors-test + storage: + spec: + mountPath: {} +storage: + spec: + mountPath: {} diff --git a/testdata/go/v2/memcached-operator/bundle/tests/scorecard/config.yaml b/testdata/go/v2/memcached-operator/bundle/tests/scorecard/config.yaml index 2ff360df3fa..8cba919ca99 100644 --- a/testdata/go/v2/memcached-operator/bundle/tests/scorecard/config.yaml +++ b/testdata/go/v2/memcached-operator/bundle/tests/scorecard/config.yaml @@ -12,6 +12,9 @@ stages: labels: suite: basic test: basic-check-spec-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-bundle-validation @@ -19,6 +22,9 @@ stages: labels: suite: olm test: olm-bundle-validation-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-crds-have-validation @@ -26,6 +32,9 @@ stages: labels: suite: olm test: olm-crds-have-validation-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-crds-have-resources @@ -33,6 +42,9 @@ stages: labels: suite: olm test: olm-crds-have-resources-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-spec-descriptors @@ -40,6 +52,9 @@ stages: labels: suite: olm test: olm-spec-descriptors-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-status-descriptors @@ -47,3 +62,9 @@ stages: labels: suite: olm test: olm-status-descriptors-test + storage: + spec: + mountPath: {} +storage: + spec: + mountPath: {} diff --git a/testdata/go/v3/memcached-operator/bundle/tests/scorecard/config.yaml b/testdata/go/v3/memcached-operator/bundle/tests/scorecard/config.yaml index 2ff360df3fa..8cba919ca99 100644 --- a/testdata/go/v3/memcached-operator/bundle/tests/scorecard/config.yaml +++ b/testdata/go/v3/memcached-operator/bundle/tests/scorecard/config.yaml @@ -12,6 +12,9 @@ stages: labels: suite: basic test: basic-check-spec-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-bundle-validation @@ -19,6 +22,9 @@ stages: labels: suite: olm test: olm-bundle-validation-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-crds-have-validation @@ -26,6 +32,9 @@ stages: labels: suite: olm test: olm-crds-have-validation-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-crds-have-resources @@ -33,6 +42,9 @@ stages: labels: suite: olm test: olm-crds-have-resources-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-spec-descriptors @@ -40,6 +52,9 @@ stages: labels: suite: olm test: olm-spec-descriptors-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-status-descriptors @@ -47,3 +62,9 @@ stages: labels: suite: olm test: olm-status-descriptors-test + storage: + spec: + mountPath: {} +storage: + spec: + mountPath: {} diff --git a/testdata/helm/memcached-operator/bundle/tests/scorecard/config.yaml b/testdata/helm/memcached-operator/bundle/tests/scorecard/config.yaml index 2ff360df3fa..8cba919ca99 100644 --- a/testdata/helm/memcached-operator/bundle/tests/scorecard/config.yaml +++ b/testdata/helm/memcached-operator/bundle/tests/scorecard/config.yaml @@ -12,6 +12,9 @@ stages: labels: suite: basic test: basic-check-spec-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-bundle-validation @@ -19,6 +22,9 @@ stages: labels: suite: olm test: olm-bundle-validation-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-crds-have-validation @@ -26,6 +32,9 @@ stages: labels: suite: olm test: olm-crds-have-validation-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-crds-have-resources @@ -33,6 +42,9 @@ stages: labels: suite: olm test: olm-crds-have-resources-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-spec-descriptors @@ -40,6 +52,9 @@ stages: labels: suite: olm test: olm-spec-descriptors-test + storage: + spec: + mountPath: {} - entrypoint: - scorecard-test - olm-status-descriptors @@ -47,3 +62,9 @@ stages: labels: suite: olm test: olm-status-descriptors-test + storage: + spec: + mountPath: {} +storage: + spec: + mountPath: {} diff --git a/website/content/en/docs/cli/operator-sdk_scorecard.md b/website/content/en/docs/cli/operator-sdk_scorecard.md index c0ed3efc8ab..7a9c65380d9 100644 --- a/website/content/en/docs/cli/operator-sdk_scorecard.md +++ b/website/content/en/docs/cli/operator-sdk_scorecard.md @@ -27,6 +27,7 @@ operator-sdk scorecard [flags] -l, --selector string label selector to determine which tests are run -s, --service-account string Service account to use for tests (default "default") -x, --skip-cleanup Disable resource cleanup after tests are run + -t, --test-output string Test output directory. (default "test-output") -w, --wait-time duration seconds to wait for tests to complete. Example: 35s (default 30s) ```