Build fixes

Makefile

@@ -9,9 +9,10 @@ go_test: install
 # just use LICENSE as a file we can harmlessly "touch" and use as a cache marker
 LICENSE: main.go pkg/*/*.go
 	go install
+	make build_setuid
 	touch LICENSE
 
-install: LICENSE build_setuid
+install: LICENSE
 
 integration_test: install
 	env BRAMBLE_INTEGRATION_TEST=truthy go test -v ./pkg/bramble/

bramble.lock

@@ -2,4 +2,5 @@
   "http://tarballs.nixos.org/stdenv-linux/x86_64/c5aabb0d603e2c1ea05f5a93b3be82437f5ebf31/bootstrap-tools.tar.xz" = "a5ce9c155ed09397614646c9717fc7cd94b1023d7b76b618d409e4fefd6e9d39"
   "https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz" = "2ae410370b8e9113968ffa6e52f38eea7f17df5f436bd6a69cc41c6ca01541a1"
   "https://brmbl.s3.amazonaws.com/patchelf.tar.gz" = "67ee6623207754a18d81624d630d9addbf6234ab1e6c44ddba9179621720f960"
+  "https://dl.google.com/go/go1.4-bootstrap-20171003.tar.gz" = "f4ff5b5eb3a3cae1c993723f3eab519c5bae18866b5e5f96fe1102f0cb5c3e52"
   "https://maxmcd.com/" = "b17315f3eac25dc23a59cc0ec2820c78a0b9890f7fea5a44deaef5a3c6cd9e59"

go.mod

@@ -6,12 +6,13 @@ require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054
 	github.com/containerd/console v1.0.0
+	github.com/creack/pty v1.1.11
+	github.com/docker/docker v1.4.2-0.20191101170500-ac7306503d23
 	github.com/fsouza/go-dockerclient v1.6.5
 	github.com/go-git/go-git/v5 v5.2.0
 	github.com/hashicorp/terraform v0.14.4
 	github.com/imdario/mergo v0.3.11 // indirect
 	github.com/jaguilar/vt100 v0.0.0-20201024211400-81de19cb81a4
-	github.com/maxmcd/gosh v0.2.1-0.20210228220323-59420ac4567a
 	github.com/mholt/archiver/v3 v3.3.1-0.20200626164424-d44471c49aa7
 	github.com/moby/moby v1.13.1
 	github.com/morikuni/aec v1.0.0
@@ -22,7 +23,7 @@ require (
 	go.starlark.net v0.0.0-20200901195727-6e684ef5eeee
 	go.uber.org/zap v1.10.0
 	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect
-	golang.org/x/sys v0.0.0-20201029080932-201ba4db2418
+	golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e
 	google.golang.org/grpc v1.34.0 // indirect
 )

go.sum

@@ -181,7 +181,10 @@ github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
+github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/daixiang0/gci v0.2.4/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4=
 github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -517,6 +520,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -570,8 +574,6 @@ github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpe
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/maxmcd/archiver/v3 v3.3.2-0.20200926140316-5fd9d38b8b8b h1:15C098O6LZ6+6bQT0aR8xsihI/E9wOnTdx4BDWsq5uY=
 github.com/maxmcd/archiver/v3 v3.3.2-0.20200926140316-5fd9d38b8b8b/go.mod h1:wZCaCDpKnb7vsqOlgW3WO756DciCRSCOZCVMkXkrxfs=
-github.com/maxmcd/gosh v0.2.1-0.20210228220323-59420ac4567a h1:3CbQNZrLMH+bskfzm3zwD8mVZzVT4a9iUqz/plrpGcQ=
-github.com/maxmcd/gosh v0.2.1-0.20210228220323-59420ac4567a/go.mod h1:/79yP5yHdovdNxLUSD8MaswCgQe7Fl1gV0Js1nLmiq8=
 github.com/maxmcd/starlark-go v0.0.0-20201021154825-b2f805d0d122 h1:5ntUQ6qQLi3wcHdxEVWud2Q5z11Pj6cQFA6OHbCvbRU=
 github.com/maxmcd/starlark-go v0.0.0-20201021154825-b2f805d0d122/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU=
 github.com/miekg/dns v1.0.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -1006,8 +1008,8 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 h1:HlFl4V6pEMziuLXyRkm5BIYq1y1GAbb02pRlWvI54OM=
-golang.org/x/sys v0.0.0-20201029080932-201ba4db2418/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b h1:ggRgirZABFolTmi3sn6Ivd9SipZwLedQ5wR0aAKnFxU=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

lib/go/build.sh

@@ -6,14 +6,22 @@ export LD_LIBRARY_PATH=$stdenv/lib
 
 mkdir -p /var/tmp
 
-cp -r $go1_4/go .
-cd ./go/src
+cp -r $go1_4/go $out
+cd $out
+include=$(pwd)/go/include
+cd $out/go/src
 ls -lah ./cmd/dist
-export GO_CCFLAGS="-I$stdenv/include-glibc"
-export CC="gcc -I$stdenv/include-glibc -Wl,-rpath=$stdenv/lib "
-export GOROOT_BOOTSTRAP=$(pwd)
+export GO_LDFLAGS="-L $stdenv/lib -I $include -I $stdenv/include-glibc -I $stdenv/include"
+export CC="gcc -L $stdenv/lib -I $include -I $stdenv/include-glibc -I $stdenv/include -Wl,-rpath=$stdenv/lib -Wl,--dynamic-linker=$stdenv/lib/ld-linux-x86-64.so.2 "
+# export GOROOT_BOOTSTRAP=$(pwd)
 export CGO_ENABLED="0"
-which sh
 sed -i 's/set -e/set -ex/g' ./make.bash
 # cat ./make.bash
 bash ./make.bash
+
+# this works for a few things, but has trouble finding the network, resolve.conf
+# syslog, no go in PATH,
+# stat testdata/libmach8db: no such file or directory
+#  stat /Users/rsc/bin/xed: no such file or directory
+# stat /usr/local/bin/arm-linux-elf-objdump: no such file or directory
+# ../bin/go test all

lib/go/default.bramble

@@ -5,11 +5,13 @@ load("github.com/maxmcd/bramble/lib")
 
 def _bootstrap():
     go1_4 = std.fetch_url("https://dl.google.com/go/go1.4-bootstrap-20171003.tar.gz")
-
+    path = "%s/bin:%s/bin" % (nix_seed.stdenv(), lib.busybox())
     return derivation(
         name="go-1.4",
         builder=lib.busybox().out + "/bin/sh",
         args=["./build.sh"],
         sources=["./build.sh"],
-        env=dict(go1_4=go1_4, stdenv=nix_seed.stdenv(), busybox=lib.busybox()),
+        env=dict(
+            go1_4=go1_4, stdenv=nix_seed.stdenv(), busybox=lib.busybox(), PATH=path
+        ),
     )

notes/06-sandboxing.md

@@ -23,8 +23,11 @@ If bramble is going to have this level of flexibility it should be sandboxed by
 - https://github.com/NixOS/nix/issues/1429 - nix disallows setuid
 - https://bazel.build/designs/2016/06/02/sandboxing.html
 Support docker as an option, would work only for linux or just for running things in docker.
+- https://unix.stackexchange.com/questions/6433/how-to-jail-a-process-without-being-root interesting notes
 
 
 ## Linux
 
-Default mounts: https://github.com/opencontainers/runtime-tools/blob/a7974a4078764ec41acf5feaa05f07854af44aa6/generate/generate.go#L174-L211
+- Default mounts: https://github.com/opencontainers/runtime-tools/blob/a7974a4078764ec41acf5feaa05f07854af44aa6/generate/generate.go#L174-L211
+- Create dev/null https://www.commandlinefu.com/commands/view/24199/create-devnull-if-accidentally-deleted-or-for-a-chroot
+- container linux with mknod examples https://github.com/cloudify-incubator/cloudify-rest-go-client/blob/f8139d8e38b0909fae3e4212eb05497483c0e5b8/container/container_linux.go

notes/06-sandboxing/minijail.sh

@@ -0,0 +1,34 @@
+set -x
+args=(
+    -T static # we want minijail to assume a static binary in case the binary can't use LD_PRELOAD
+    # -U # new user namespace
+    -C /tmp/bramble-chroot-713833583 # our chroot location
+    # -N # new cgroup namespace
+    # -p # new pid namespace
+    --mount-dev # mount a minimal dev
+    -u maxm  # set user id
+    -e
+    -g users # set group id
+    # -v # new vfs namespace ???
+    -b /home/maxm/bramble/bramble_store_padding/bramble_
+    -b /home/maxm/bramble/bramble_store_padding/bramble_/bramble_build_directory121331829 1
+    -b /home/maxm/bramble/bramble_store_padding/bramble_/bramble_build_directory463911248 1
+    /home/maxm/bramble/bramble_store_padding/bramble_/ieqjuyrfv7lmdb2bur76jgjcrd33j7na/bin/sh
+)
+mkdir -p /tmp/bramble-chroot-713833583/proc
+mkdir -p /tmp/bramble-chroot-713833583/dev
+mkdir -p /tmp/bramble-chroot-713833583/var
+# use this args hack so that we can comment
+sudo strace minijail0 "${args[@]}"
+
+
+# this doesn't work, seems hung up on directory permissions unless
+# I make my homedir readable
+# even after that it exists with a "file not found" error that I don't understand
+
+# with strace it seems to be having trouble finding a library:
+# ¯\_(ツ)_/¯
+#
+# stat("/nix/store/gafigwfaimlziam6qhw1m8dz4h952g1n-glibc-2.32-35/lib/x86_64", 0x7ffd90b68830) = -1 ENOENT (No such file or directory)
+# openat(AT_FDCWD, "/nix/store/ah4h9wpsz8yvrfnlk7ldm97q131b1di7-libcap-2.46-lib/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
+# wait4(27918, libminijail[27918]: execve(1) failed: No such file or directory

pkg/bramble/bramble.go

@@ -132,7 +132,7 @@ func (b *Bramble) buildDerivationIfNew(ctx context.Context, drv *Derivation) (er
 	logger.Print("Building derivation", filename)
 	logger.Debugw(drv.prettyJSON())
 
-	if err = b.buildDerivation(ctx, drv); err != nil {
+	if err = b.buildDerivation(ctx, drv, false); err != nil {
 		return errors.Wrap(err, "error building "+filename)
 	}
 	return b.writeDerivation(drv)
@@ -161,7 +161,7 @@ func (b *Bramble) hashAndMoveFetchURL(ctx context.Context, drv *Derivation, outp
 	return
 }
 
-func (b *Bramble) buildDerivation(ctx context.Context, drv *Derivation) (err error) {
+func (b *Bramble) buildDerivation(ctx context.Context, drv *Derivation, shell bool) (err error) {
 	var task *trace.Task
 	ctx, task = trace.NewTask(ctx, "buildDerivation")
 	defer task.End()
@@ -187,13 +187,18 @@ func (b *Bramble) buildDerivation(ctx context.Context, drv *Derivation) (err err
 	if err != nil {
 		return
 	}
+
+	if shell && (drv.Builder == "fetch_url" || drv.Builder == "fetch_git") {
+		return errors.New("can't spawn a shell with a builtin builder")
+	}
+
 	switch drv.Builder {
 	case "fetch_url":
 		err = b.fetchURLBuilder(ctx, drvCopy, outputPaths)
 	case "fetch_git":
 		err = b.fetchGitBuilder(ctx, drvCopy, outputPaths)
 	default:
-		err = b.regularBuilder(ctx, drvCopy, buildDir, outputPaths)
+		err = b.regularBuilder(ctx, drvCopy, buildDir, outputPaths, shell)
 	}
 	if err != nil {
 		return
@@ -505,7 +510,8 @@ func (b *Bramble) dockerRegularBuilder(ctx context.Context, drv *Derivation, bui
 	return b.runDockerBuild(ctx, drv.filename(), options)
 }
 
-func (b *Bramble) regularBuilder(ctx context.Context, drv *Derivation, buildDir string, outputPaths map[string]string) (err error) {
+func (b *Bramble) regularBuilder(ctx context.Context, drv *Derivation, buildDir string,
+	outputPaths map[string]string, shell bool) (err error) {
 	builderLocation := drv.Builder
 	if _, err := os.Stat(builderLocation); err != nil {
 		return errors.Wrap(err, "builder location doesn't exist")
@@ -534,7 +540,6 @@ func (b *Bramble) regularBuilder(ctx context.Context, drv *Derivation, buildDir
 	sbx := sandbox.Sandbox{
 		Path:       builderLocation,
 		Args:       drv.Args,
-		Stdin:      os.Stdin,
 		Stdout:     os.Stdout,
 		Stderr:     os.Stderr,
 		UserID:     uid,
@@ -544,6 +549,10 @@ func (b *Bramble) regularBuilder(ctx context.Context, drv *Derivation, buildDir
 		Dir:        filepath.Join(buildDir, drv.BuildContextRelativePath),
 		Mounts:     mounts,
 	}
+	if shell {
+		sbx.Args = nil
+		sbx.Stdin = os.Stdin
+	}
 	return sbx.Run(ctx)
 }
 
@@ -748,7 +757,7 @@ func (b *Bramble) assembleDerivationDependencyGraph(dos DerivationOutputs) *Acyc
 	return graph
 }
 
-func (b *Bramble) buildDerivationOutputs(dos DerivationOutputs) (err error) {
+func (b *Bramble) buildDerivationOutputs(ctx context.Context, dos DerivationOutputs, skipDerivation *Derivation) (err error) {
 	graph := b.assembleDerivationDependencyGraph(dos)
 	var wg sync.WaitGroup
 	errChan := make(chan error)
@@ -757,7 +766,7 @@ func (b *Bramble) buildDerivationOutputs(dos DerivationOutputs) (err error) {
 	if err = graph.Validate(); err != nil {
 		return err
 	}
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(ctx)
 	go func() {
 		graph.Walk(func(v dag.Vertex) (_ tfdiags.Diagnostics) {
 			if errored {
@@ -777,9 +786,13 @@ func (b *Bramble) buildDerivationOutputs(dos DerivationOutputs) (err error) {
 			if drv.Output(do.OutputName).Path != "" {
 				return
 			}
+			if skipDerivation != nil && skipDerivation.filename() == drv.filename() {
+				return
+			}
 			wg.Add(1)
 			if err := b.buildDerivationIfNew(ctx, drv); err != nil {
-				// Passing the error might block, so we need an explicit Done call here.
+				// Passing the error might block, so we need an explicit Done
+				// call here.
 				wg.Done()
 				errored = true
 				logger.Print(err)
@@ -794,7 +807,8 @@ func (b *Bramble) buildDerivationOutputs(dos DerivationOutputs) (err error) {
 	err = <-errChan
 	cancel() // Call cancel on the context, no-op if nothing is running
 	if err != nil {
-		// If we receive an error cancel the context and wait for any jobs that are running.
+		// If we receive an error cancel the context and wait for any jobs that
+		// are running.
 		wg.Wait()
 	}
 	return err
@@ -975,10 +989,11 @@ func (b *Bramble) repl(_ []string) (err error) {
 	return nil
 }
 
-func (b *Bramble) build(args []string) (err error) {
+func (b *Bramble) parseBuildArg(cmd string, args []string) (module, fn string, derivations []*Derivation, err error) {
 	if len(args) == 0 {
-		logger.Print(`"bramble build" requires 1 argument`)
-		return flag.ErrHelp
+		logger.Printfln(`"bramble %s" requires 1 argument`, cmd)
+		err = flag.ErrHelp
+		return
 	}
 
 	if err = b.init(); err != nil {
@@ -987,8 +1002,7 @@ func (b *Bramble) build(args []string) (err error) {
 
 	// parse something like ./tests:foo into the correct module and function
 	// name
-	module, fn, err := b.parseModuleFuncArgument(args)
-	if err != nil {
+	if module, fn, err = b.parseModuleFuncArgument(args); err != nil {
 		return
 	}
 
@@ -999,23 +1013,58 @@ func (b *Bramble) build(args []string) (err error) {
 	}
 	toCall, ok := globals[fn]
 	if !ok {
-		return errors.Errorf("function %q not found in module %q", fn, module)
+		err = errors.Errorf("function %q not found in module %q", fn, module)
+		return
 	}
 
 	values, err := starlark.Call(&starlark.Thread{}, toCall, nil, nil)
 	if err != nil {
-		return errors.Wrap(err, "error running")
+		err = errors.Wrap(err, "error running")
+		return
 	}
 
 	// The function must return a single derivation or a list of derivations, or
 	// a tuple of derivations. We turn them into an array.
-	returnedDerivations := valuesToDerivations(values)
+	derivations = valuesToDerivations(values)
+	return
+}
+
+func (b *Bramble) shell(ctx context.Context, args []string) (err error) {
+	module, fn, derivations, err := b.parseBuildArg("build", args)
+	if err != nil {
+		return err
+	}
+	if len(derivations) > 1 {
+		return errors.New(`cannot run "bramble shell" with a function that returns multiple derivations`)
+	}
+	shellDerivation := derivations[0]
+
+	if err = b.buildDerivationOutputs(ctx, b.derivationsToDerivationOutputs(derivations), shellDerivation); err != nil {
+		return
+	}
 
-	if err = b.buildDerivationOutputs(b.derivationsToDerivationOutputs(returnedDerivations)); err != nil {
+	if err := b.writeConfigMetadata(derivations, module, fn); err != nil {
+		return err
+	}
+	filename := shellDerivation.filename()
+	logger.Print("Launching shell for derivation", filename)
+	logger.Debugw(shellDerivation.prettyJSON())
+	if err = b.buildDerivation(ctx, shellDerivation, true); err != nil {
+		return errors.Wrap(err, "error spawning "+filename)
+	}
+	return nil
+}
+
+func (b *Bramble) build(ctx context.Context, args []string) (err error) {
+	module, fn, derivations, err := b.parseBuildArg("build", args)
+	if err != nil {
+		return err
+	}
+	if err = b.buildDerivationOutputs(ctx, b.derivationsToDerivationOutputs(derivations), nil); err != nil {
 		return
 	}
 
-	return b.writeConfigMetadata(returnedDerivations, fn, module)
+	return b.writeConfigMetadata(derivations, module, fn)
 }
 
 func valuesToDerivations(values starlark.Value) (derivations []*Derivation) {