Skip to content

Commit aeec951

Browse files
sreyasreya
committed
add integration test
1 parent 1e5fbce commit aeec951

File tree

6 files changed

+97
-62
lines changed

6 files changed

+97
-62
lines changed

‎cli/docker.go

+23-46
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,12 @@ import (
77
"io"
88
"net/url"
99
"os"
10-
"os/signal"
10+
"os/exec"
1111
"path"
1212
"path/filepath"
1313
"sort"
1414
"strconv"
1515
"strings"
16-
"syscall"
17-
"time"
1816

1917
dockertypes "github.com/docker/docker/api/types"
2018
"github.com/docker/docker/api/types/container"
@@ -148,7 +146,7 @@ type flags struct {
148146
ethlink string
149147
}
150148

151-
func dockerCmd() *cobra.Command {
149+
func dockerCmd(ch chan func() error) *cobra.Command {
152150
var flags flags
153151

154152
cmd := &cobra.Command{
@@ -287,7 +285,7 @@ func dockerCmd() *cobra.Command {
287285
return xerrors.Errorf("wait for dockerd: %w", err)
288286
}
289287

290-
err = runDockerCVM(ctx, log, client, blog, flags)
288+
err = runDockerCVM(ctx, log, client, blog, ch, flags)
291289
if err != nil {
292290
// It's possible we failed because we ran out of disk while
293291
// pulling the image. We should restart the daemon and use
@@ -316,7 +314,7 @@ func dockerCmd() *cobra.Command {
316314
}()
317315

318316
log.Debug(ctx, "reattempting container creation")
319-
err = runDockerCVM(ctx, log, client, blog, flags)
317+
err = runDockerCVM(ctx, log, client, blog, ch, flags)
320318
}
321319
if err != nil {
322320
blog.Errorf("Failed to run envbox: %v", err)
@@ -359,7 +357,7 @@ func dockerCmd() *cobra.Command {
359357
return cmd
360358
}
361359

362-
func runDockerCVM(ctx context.Context, log slog.Logger, client dockerutil.DockerClient, blog buildlog.Logger, flags flags) error {
360+
func runDockerCVM(ctx context.Context, log slog.Logger, client dockerutil.DockerClient, blog buildlog.Logger, shutdownCh chan func() error, flags flags) error {
363361
fs := xunix.GetFS(ctx)
364362

365363
// Set our OOM score to something really unfavorable to avoid getting killed
@@ -692,6 +690,7 @@ func runDockerCVM(ctx context.Context, log slog.Logger, client dockerutil.Docker
692690
if err != nil {
693691
return xerrors.Errorf("create exec: %w", err)
694692
}
693+
695694
resp, err := client.ContainerExecAttach(ctx, bootstrapExec.ID, dockertypes.ExecStartCheck{})
696695
if err != nil {
697696
return xerrors.Errorf("attach exec: %w", err)
@@ -715,53 +714,31 @@ func runDockerCVM(ctx context.Context, log slog.Logger, client dockerutil.Docker
715714
}
716715
}()
717716

718-
inspect, err := client.ContainerExecInspect(ctx, bootstrapExec.ID)
717+
// We can't just call ExecInspect because there's a race where the cmd
718+
// hasn't been assigned a PID yet.
719+
bootstrapPID, err := dockerutil.GetExecPID(ctx, client, bootstrapExec.ID)
719720
if err != nil {
720721
return xerrors.Errorf("exec inspect: %w", err)
721722
}
722-
sigs := make(chan os.Signal, 1)
723-
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
724-
go func() {
725-
err := func() error {
726-
log.Info(ctx, "waiting for signal")
727-
sig := <-sigs
728-
sigstr := "TERM"
729-
if sig == syscall.SIGINT {
730-
sigstr = "INT"
731-
}
732-
log.Debug(ctx, "received signal", slog.F("signal", sigstr))
733723

734-
killExec, err := client.ContainerExecCreate(ctx, containerID, dockertypes.ExecConfig{
735-
User: imgMeta.UID,
736-
Cmd: []string{"sh", "-c", fmt.Sprintf("kill -%s %d", sigstr, inspect.Pid)},
737-
AttachStdout: true,
738-
AttachStderr: true,
739-
})
740-
if err != nil {
741-
return xerrors.Errorf("create kill exec: %w", err)
742-
}
743-
744-
err = dockerutil.WaitForExit(ctx, client, killExec.ID)
745-
if err != nil {
746-
return xerrors.Errorf("wait for kill exec to complete: %w", err)
747-
}
724+
shutdownCh <- func() error {
725+
log.Debug(ctx, "killing container", slog.F("bootstrap_pid", bootstrapPID))
748726

749-
err = dockerutil.WaitForExit(ctx, client, bootstrapExec.ID)
750-
if err != nil {
751-
return xerrors.Errorf("wait for exit: %w", err)
752-
}
727+
// The PID returned is the PID _outside_ the container...
728+
out, err := exec.Command("kill", "-TERM", strconv.Itoa(bootstrapPID)).CombinedOutput()
729+
if err != nil {
730+
return xerrors.Errorf("kill bootstrap process (%s): %w", out, err)
731+
}
753732

754-
return nil
755-
}()
756-
log.Info(ctx, "exiting envbox", slog.Error(err))
757-
log.Sync()
733+
log.Debug(ctx, "sent kill signal waiting for process to exit")
734+
err = dockerutil.WaitForExit(ctx, client, bootstrapExec.ID)
758735
if err != nil {
759-
os.Exit(1)
736+
return xerrors.Errorf("wait for exit: %w", err)
760737
}
761-
os.Exit(0)
762-
}()
763-
log.Info(ctx, "HELP")
764-
time.Sleep(time.Second)
738+
739+
log.Debug(ctx, "bootstrap process successfully exited")
740+
return nil
741+
}
765742

766743
return nil
767744
}

‎cli/root.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"github.com/spf13/cobra"
55
)
66

7-
func Root() *cobra.Command {
7+
func Root(ch chan func() error) *cobra.Command {
88
cmd := &cobra.Command{
99
Use: "envbox",
1010
SilenceErrors: true,
@@ -15,6 +15,6 @@ func Root() *cobra.Command {
1515
},
1616
}
1717

18-
cmd.AddCommand(dockerCmd())
18+
cmd.AddCommand(dockerCmd(ch))
1919
return cmd
2020
}

‎cmd/envbox/main.go

+23-4
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,37 @@ package main
33
import (
44
"fmt"
55
"os"
6+
"os/signal"
67
"runtime"
8+
"syscall"
79

810
"github.com/coder/envbox/cli"
911
)
1012

1113
func main() {
12-
_, err := cli.Root().ExecuteC()
14+
ch := make(chan func() error, 1)
15+
sigs := make(chan os.Signal, 1)
16+
signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT, syscall.SIGWINCH)
17+
go func() {
18+
fmt.Println("waiting for signal")
19+
<-sigs
20+
fmt.Println("Got signal")
21+
select {
22+
case fn := <-ch:
23+
fmt.Println("running shutdown function")
24+
err := fn()
25+
if err != nil {
26+
fmt.Fprintf(os.Stderr, "shutdown function failed: %v", err)
27+
os.Exit(1)
28+
}
29+
default:
30+
fmt.Println("no shutdown function")
31+
}
32+
os.Exit(0)
33+
}()
34+
_, err := cli.Root(ch).ExecuteC()
1335
if err != nil {
14-
_, _ = fmt.Fprintln(os.Stderr, err.Error())
1536
os.Exit(1)
1637
}
17-
18-
// We exit the main thread while keepin all the other procs goin strong.
1938
runtime.Goexit()
2039
}

‎dockerutil/exec.go

+17-1
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,23 @@ func ExecContainer(ctx context.Context, client DockerClient, config ExecConfig)
9595
return &buf, inspect.Pid, nil
9696
}
9797

98+
func GetExecPID(ctx context.Context, client DockerClient, execID string) (int, error) {
99+
for r := retry.New(time.Second, time.Second); r.Wait(ctx); {
100+
inspect, err := client.ContainerExecInspect(ctx, execID)
101+
if err != nil {
102+
return 0, xerrors.Errorf("exec inspect: %w", err)
103+
}
104+
105+
if inspect.Pid == 0 {
106+
continue
107+
}
108+
return inspect.Pid, nil
109+
}
110+
111+
return 0, ctx.Err()
112+
113+
}
114+
98115
func WaitForExit(ctx context.Context, client DockerClient, execID string) error {
99116
for r := retry.New(time.Second, time.Second); r.Wait(ctx); {
100117
inspect, err := client.ContainerExecInspect(ctx, execID)
@@ -112,6 +129,5 @@ func WaitForExit(ctx context.Context, client DockerClient, execID string) error
112129

113130
return nil
114131
}
115-
116132
return ctx.Err()
117133
}

‎integration/docker_test.go

+6-8
Original file line numberDiff line numberDiff line change
@@ -289,10 +289,12 @@ func TestDocker(t *testing.T) {
289289

290290
binds = append(binds, bindMount(homeDir, "/home/coder", false))
291291

292+
envs := []string{fmt.Sprintf("%s=%s:%s", cli.EnvMounts, "/home/coder", "/home/coder")}
292293
// Run the envbox container.
293294
resource := integrationtest.RunEnvbox(t, pool, &integrationtest.CreateDockerCVMConfig{
294295
Image: integrationtest.UbuntuImage,
295296
Username: "root",
297+
Envs: envs,
296298
Binds: binds,
297299
BootstrapScript: sigtrapScript,
298300
})
@@ -303,9 +305,8 @@ func TestDocker(t *testing.T) {
303305
})
304306
require.Error(t, err)
305307

306-
time.Sleep(time.Second * 5)
307-
err = pool.Client.StopContainer(resource.Container.ID, 30)
308-
require.NoError(t, err)
308+
// Simulate a shutdown.
309+
integrationtest.StopContainer(t, pool, resource.Container.ID, 30*time.Second)
309310

310311
err = resource.Close()
311312
require.NoError(t, err)
@@ -315,6 +316,7 @@ func TestDocker(t *testing.T) {
315316
resource = integrationtest.RunEnvbox(t, pool, &integrationtest.CreateDockerCVMConfig{
316317
Image: integrationtest.UbuntuImage,
317318
Username: "root",
319+
Envs: envs,
318320
Binds: binds,
319321
BootstrapScript: sigtrapScript,
320322
})
@@ -357,17 +359,13 @@ func bindMount(src, dest string, ro bool) string {
357359
}
358360

359361
const sigtrapScript = `#!/bin/bash
360-
361-
# Function to handle cleanup
362362
cleanup() {
363-
touch /home/coder/foo
363+
echo "HANDLING A SIGNAL!" && touch /home/coder/foo && echo "touched file"
364364
exit 0
365365
}
366366
367-
# Trap SIGINT (Ctrl+C) and SIGTERM signals
368367
trap 'cleanup' INT TERM
369368
370-
# Main loop or processing logic (replace with your script's logic)
371369
while true; do
372370
echo "Working..."
373371
sleep 1

‎integration/integrationtest/docker.go

+26-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ const (
3333
HelloWorldImage = "gcr.io/coder-dev-1/sreya/hello-world"
3434
// UbuntuImage is just vanilla ubuntu (80MB) but the user is set to a non-root
3535
// user .
36-
UbuntuImage = "gcr.io/coder-dev-1/sreya/ubuntu-coder"
36+
UbuntuImage = "gcr.io/coder-dev-1/sreya/ubuntu-coder:jon"
3737
)
3838

3939
// TODO use df to determine if an environment is running in a docker container or not.
@@ -264,6 +264,31 @@ func ExecEnvbox(t *testing.T, pool *dockertest.Pool, conf ExecConfig) ([]byte, e
264264
return buf.Bytes(), nil
265265
}
266266

267+
func StopContainer(t *testing.T, pool *dockertest.Pool, id string, to time.Duration) {
268+
t.Helper()
269+
270+
err := pool.Client.KillContainer(docker.KillContainerOptions{
271+
ID: id,
272+
Signal: docker.SIGTERM,
273+
})
274+
require.NoError(t, err)
275+
276+
ctx, cancel := context.WithTimeout(context.Background(), to)
277+
defer cancel()
278+
for r := retry.New(time.Second, time.Second); r.Wait(ctx); {
279+
cnt, err := pool.Client.InspectContainer(id)
280+
require.NoError(t, err)
281+
282+
if cnt.State.Running {
283+
continue
284+
}
285+
286+
return
287+
}
288+
289+
t.Fatalf("timed out waiting for container %s to stop", id)
290+
}
291+
267292
// cmdLineEnvs returns args passed to the /envbox command
268293
// but using their env var alias.
269294
func cmdLineEnvs(c *CreateDockerCVMConfig) []string {

0 commit comments

Comments
 (0)