Skip to content

Commit

Permalink
kola: Add run-ext-bin command
Browse files Browse the repository at this point in the history
Related: coreos#1159

I want to start converting some of the ostree/rpm-ostree
tests to using kola more "natively".  This is part of that; the
`run-ext-bin` command injects a single binary into the target
system, driving it via a systemd unit to completion.

There are further elaborations of this, including injecting
a full container, etc.  (That is cheap in the local qemu
case particularly once we have virtio-fs)
  • Loading branch information
cgwalters committed Mar 11, 2020
1 parent dc88a07 commit 267cba5
Show file tree
Hide file tree
Showing 5 changed files with 351 additions and 0 deletions.
36 changes: 36 additions & 0 deletions mantle/cmd/kola/kola.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,22 @@ will be ignored.
SilenceUsage: true,
}

cmdRunExtBin = &cobra.Command{
Use: "run-ext-bin",
Short: "Run an external test (single binary)",
Long: `Run an externally defined test
This injects a single binary into the target system and executes it as a systemd
unit. The test is considered successful when the service exits normally, and
failed if the test exits non-zero - but the service being killed by e.g. SIGTERM
is ignored. This is intended to allow rebooting the system.
`,

Args: cobra.ExactArgs(1),
PreRunE: preRun,
RunE: runRunExtBin,
}

cmdHttpServer = &cobra.Command{
Use: "http-server",
Short: "Run a static webserver",
Expand Down Expand Up @@ -122,10 +138,14 @@ This can be useful for e.g. serving locally built OSTree repos to qemu.
findParentImage bool
qemuImageDir string
qemuImageDirIsTemp bool

extDependencyDir string
)

func init() {
root.AddCommand(cmdRun)
root.AddCommand(cmdRunExtBin)
cmdRunExtBin.Flags().StringVar(&extDependencyDir, "depdir", "", "Copy (rsync) dir to target, available as ${KOLA_EXT_DATA}")

root.AddCommand(cmdList)
cmdList.Flags().BoolVar(&listJSON, "json", false, "format output in JSON")
Expand Down Expand Up @@ -190,6 +210,22 @@ func runRun(cmd *cobra.Command, args []string) error {
return runErr
}

func runRunExtBin(cmd *cobra.Command, args []string) error {
extbin := args[0]

outputDir, err := kola.SetupOutputDir(outputDir, kolaPlatform)
if err != nil {
return err
}

runErr := kola.RunExtBin(kolaPlatform, outputDir, extbin, extDependencyDir)
if err := writeProps(); err != nil {
return errors.Wrapf(err, "writing properties")
}

return runErr
}

func writeProps() error {
f, err := os.OpenFile(filepath.Join(outputDir, "properties.json"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
if err != nil {
Expand Down
116 changes: 116 additions & 0 deletions mantle/cmd/kolet/kolet.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,14 @@
package main

import (
"fmt"
"os"
"strings"
"time"

systemddbus "github.com/coreos/go-systemd/v22/dbus"
"github.com/coreos/pkg/capnslog"
"github.com/pkg/errors"
"github.com/spf13/cobra"

"github.com/coreos/mantle/cli"
Expand All @@ -27,6 +32,12 @@ import (
_ "github.com/coreos/mantle/kola/registry"
)

const (
// From /usr/include/bits/siginfo-consts.h
CLD_EXITED int32 = 1
CLD_KILLED int32 = 2
)

var (
plog = capnslog.NewPackageLogger("github.com/coreos/mantle", "kolet")

Expand All @@ -41,6 +52,13 @@ var (
Short: "Run a given test's native function",
Run: run,
}

cmdRunExtUnit = &cobra.Command{
Use: "run-test-unit [unitname]",
Short: "Monitor execution of a systemd unit",
RunE: runExtUnit,
SilenceUsage: true,
}
)

func run(cmd *cobra.Command, args []string) {
Expand Down Expand Up @@ -80,10 +98,108 @@ func registerTestMap(m map[string]*register.Test) {
}
}

// dispatchRunExtUnit returns true if unit completed successfully, false if
// it's still running (or unit was terminated by SIGTERM)
func dispatchRunExtUnit(unitname string, sdconn *systemddbus.Conn) (bool, error) {
props, err := sdconn.GetAllProperties(unitname)
if err != nil {
return false, errors.Wrapf(err, "listing unit properties")
}

result := props["Result"]
if result == "exit-code" {
return false, fmt.Errorf("Unit %s exited with code %d", unitname, props["ExecMainStatus"])
}

state := props["ActiveState"]
substate := props["SubState"]

switch state {
case "inactive":
fmt.Printf("Starting %s\n", unitname)
sdconn.StartUnit(unitname, "fail", nil)
return false, nil
case "activating":
return false, nil
case "active":
{
switch substate {
case "exited":
maincode := props["ExecMainCode"]
mainstatus := props["ExecMainStatus"]
switch maincode {
case CLD_EXITED:
if mainstatus == int32(0) {
return true, nil
} else {
// I don't think this can happen, we'd have exit-code above; but just
// for completeness
return false, fmt.Errorf("Unit %s failed with code %d", unitname, mainstatus)
}
case CLD_KILLED:
// SIGTERM; we explicitly allow that, expecting we're rebooting.
if mainstatus == 15 {
return false, nil
} else {
return true, fmt.Errorf("Unit %s killed by signal %d", unitname, mainstatus)
}
default:
return false, fmt.Errorf("Unit %s had unhandled code %d", unitname, maincode)
}
case "running":
return false, nil
case "failed":
return true, fmt.Errorf("Unit %s in substate 'failed'", unitname)
default:
// Pass through other states
return false, nil
}
}
default:
return false, fmt.Errorf("Unhandled systemd unit state %s", state)
}
}

func runExtUnit(cmd *cobra.Command, args []string) error {
unitname := args[0]
if strings.Index(unitname, ".") < 0 {
unitname = unitname + ".service"
}
sdconn, err := systemddbus.NewSystemConnection()
if err != nil {
return errors.Wrapf(err, "systemd connection")
}
if err := sdconn.Subscribe(); err != nil {
return err
}
dispatchRunExtUnit(unitname, sdconn)
unitevents, uniterrs := sdconn.SubscribeUnits(time.Second)

for {
select {
case m := <-unitevents:
for n := range m {
if n == unitname {
r, err := dispatchRunExtUnit(unitname, sdconn)
if err != nil {
return err
}
if r {
return nil
}
}
}
case m := <-uniterrs:
return m
}
}
}

func main() {
registerTestMap(register.Tests)
registerTestMap(register.UpgradeTests)
root.AddCommand(cmdRun)
root.AddCommand(cmdRunExtUnit)

cli.Execute(root)
}
1 change: 1 addition & 0 deletions mantle/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ require (
github.com/coreos/container-linux-config-transpiler v0.8.0
github.com/coreos/coreos-cloudinit v1.11.0
github.com/coreos/go-semver v0.3.0
github.com/coreos/go-systemd/v22 v22.0.0
github.com/coreos/ign-converter v0.0.0-20200228175238-237c8512310a
github.com/coreos/ignition v0.35.0
github.com/coreos/ignition/v2 v2.1.1
Expand Down
145 changes: 145 additions & 0 deletions mantle/kola/runext.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
// Copyright 2020 Red Hat, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kola

import (
"encoding/json"
"fmt"
"os"
"path/filepath"

ignconverter "github.com/coreos/ign-converter"
ignv3types "github.com/coreos/ignition/v2/config/v3_0/types"
"github.com/kballard/go-shellquote"
"github.com/pkg/errors"

"github.com/coreos/mantle/platform"
"github.com/coreos/mantle/platform/conf"
"github.com/coreos/mantle/util"
)

// kolaExtBinDataDir is where data will be stored
const kolaExtBinDataDir = "/var/opt/kola/extdata"

// kolaExtBinDataEnv is an environment variable pointing to the above
const kolaExtBinDataEnv = "KOLA_EXT_DATA"

func RunExtBin(pltfrm, outputDir, extbin string, extdata string) error {
if CosaBuild == nil {
return fmt.Errorf("Must specify --cosa-build")
}

plog.Debugf("Creating flight")
flight, err := NewFlight(pltfrm)
if err != nil {
return errors.Wrapf(err, "Creating flight")
}
defer flight.Destroy()

rconf := &platform.RuntimeConfig{
OutputDir: outputDir,
}
plog.Debugf("Creating cluster")
c, err := flight.NewCluster(rconf)
if err != nil {
return err
}

unitname := "kola-runext.service"
remotepath := fmt.Sprintf("/usr/local/bin/kola-runext-%s", filepath.Base(extbin))
unit := fmt.Sprintf(`[Unit]
[Service]
Type=oneshot
RemainAfterExit=yes
Environment=%s=%s
ExecStart=%s
[Install]
RequiredBy=multi-user.target
`, kolaExtBinDataEnv, kolaExtBinDataDir, remotepath)
config := ignv3types.Config{
Ignition: ignv3types.Ignition{
Version: "3.0.0",
},
Systemd: ignv3types.Systemd{
Units: []ignv3types.Unit{
{
Name: unitname,
Contents: &unit,
Enabled: util.BoolToPtr(false),
},
},
},
}

var serializedConfig []byte
if Options.IgnitionVersion == "v2" {
ignc2, err := ignconverter.Translate3to2(config)
if err != nil {
return err
}
buf, err := json.Marshal(ignc2)
if err != nil {
return err
}
serializedConfig = buf
} else {
buf, err := json.Marshal(config)
if err != nil {
return err
}
serializedConfig = buf
}

plog.Debugf("Creating machine")
mach, err := c.NewMachine(conf.Ignition(string(serializedConfig)))
if err != nil {
return err
}

machines := []platform.Machine{mach}
scpKolet(machines, architecture(pltfrm))
{
in, err := os.Open(extbin)
if err != nil {
return err
}
defer in.Close()
if err := platform.InstallFile(in, mach, remotepath); err != nil {
return errors.Wrapf(err, "uploading %s", extbin)
}
}

if extdata != "" {
if err := platform.CopyDirToMachine(extdata, mach, kolaExtBinDataDir); err != nil {
return err
}
}

plog.Debugf("Running kolet")
_, stderr, err := mach.SSH(fmt.Sprintf("sudo ./kolet run-test-unit %s", shellquote.Join(unitname)))
if err != nil {
fmt.Println("External test failed, attempting to gather status")
out, _, _ := mach.SSH(fmt.Sprintf("sudo systemctl status %s", shellquote.Join(unitname)))
fmt.Printf("%s\n", string(out))
if Options.SSHOnTestFailure {
plog.Errorf("dropping to shell: kolet failed: %v: %s", err, stderr)
platform.Manhole(mach)
} else {
return errors.Wrapf(err, "kolet failed: %s", stderr)
}
}

return nil
}
Loading

0 comments on commit 267cba5

Please sign in to comment.