Ensuring config bools are honored, porcelain tweaks for docker

src/github.com/travis-ci/worker/backend/docker.go

@@ -30,6 +30,8 @@ type DockerInstance struct {
 	client    *docker.Client
 	provider  *DockerProvider
 	container *docker.Container
+
+	imageName string
 }
 
 func NewDockerProvider(cfg *config.ProviderConfig) (*DockerProvider, error) {
@@ -79,7 +81,7 @@ func (p *DockerProvider) Start(ctx gocontext.Context, startAttributes *StartAttr
 		return nil, err
 	}
 
-	imageID, err := p.imageForLanguage(startAttributes.Language)
+	imageID, imageName, err := p.imageForLanguage(startAttributes.Language)
 	if err != nil {
 		return nil, err
 	}
@@ -150,6 +152,7 @@ func (p *DockerProvider) Start(ctx gocontext.Context, startAttributes *StartAttr
 			client:    p.client,
 			provider:  p,
 			container: container,
+			imageName: imageName,
 		}, nil
 	case err := <-errChan:
 		return nil, err
@@ -162,10 +165,10 @@ func (p *DockerProvider) Start(ctx gocontext.Context, startAttributes *StartAttr
 
 }
 
-func (p *DockerProvider) imageForLanguage(language string) (string, error) {
+func (p *DockerProvider) imageForLanguage(language string) (string, string, error) {
 	images, err := p.client.ListImages(docker.ListImagesOptions{All: true})
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 
 	for _, image := range images {
@@ -177,13 +180,13 @@ func (p *DockerProvider) imageForLanguage(language string) (string, error) {
 		} {
 			for _, tag := range image.RepoTags {
 				if tag == searchTag {
-					return image.ID, nil
+					return image.ID, tag, nil
 				}
 			}
 		}
 	}
 
-	return "", fmt.Errorf("no image found with language %s", language)
+	return "", "", fmt.Errorf("no image found with language %s", language)
 }
 
 func (p *DockerProvider) checkoutCPUSets() (string, error) {
@@ -325,5 +328,5 @@ func (i *DockerInstance) ID() string {
 		return "{unidentified}"
 	}
 
-	return fmt.Sprintf("%s:%s", i.container.ID, i.container.Image)
+	return fmt.Sprintf("%s:%s", i.container.ID[0:7], i.imageName)
 }

src/github.com/travis-ci/worker/build_script_generator.go

@@ -12,7 +12,7 @@ import (
 	"github.com/bitly/go-simplejson"
 	"github.com/travis-ci/worker/config"
 	"github.com/travis-ci/worker/metrics"
-	"golang.org/x/net/context"
+	gocontext "golang.org/x/net/context"
 )
 
 // A BuildScriptGeneratorError is sometimes used by the Generate method on a
@@ -26,7 +26,7 @@ type BuildScriptGeneratorError struct {
 
 // A BuildScriptGenerator generates a build script for a given job payload.
 type BuildScriptGenerator interface {
-	Generate(context.Context, *simplejson.Json) ([]byte, error)
+	Generate(gocontext.Context, *simplejson.Json) ([]byte, error)
 }
 
 type webBuildScriptGenerator struct {
@@ -81,7 +81,7 @@ func NewBuildScriptGenerator(cfg *config.Config) BuildScriptGenerator {
 	}
 }
 
-func (g *webBuildScriptGenerator) Generate(ctx context.Context, payload *simplejson.Json) ([]byte, error) {
+func (g *webBuildScriptGenerator) Generate(ctx gocontext.Context, payload *simplejson.Json) ([]byte, error) {
 	if g.aptCacheHost != "" {
 		payload.SetPath([]string{"hosts", "apt_cache"}, g.aptCacheHost)
 	}
@@ -92,6 +92,7 @@ func (g *webBuildScriptGenerator) Generate(ctx context.Context, payload *simplej
 	payload.Set("paranoid", g.paranoid)
 	payload.Set("fix_resolv_conf", g.fixResolvConf)
 	payload.Set("fix_etc_hosts", g.fixEtcHosts)
+
 	if g.cacheType != "" {
 		payload.SetPath([]string{"cache_options", "type"}, g.cacheType)
 		payload.SetPath([]string{"cache_options", "fetch_timeout"}, g.cacheFetchTimeout)

src/github.com/travis-ci/worker/cmd/travis-worker/main.go

@@ -70,23 +70,23 @@ func runWorker(c *cli.Context) {
 	if cfg.SentryDSN != "" {
 		sentryHook, err := logrus_sentry.NewSentryHook(cfg.SentryDSN, []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel})
 		if err != nil {
-			context.LoggerFromContext(ctx).WithField("err", err).Error("couldn't create sentry hook")
+			logger.WithField("err", err).Error("couldn't create sentry hook")
 		}
 
 		logrus.AddHook(sentryHook)
 	}
 
 	if cfg.LibratoEmail != "" && cfg.LibratoToken != "" && cfg.LibratoSource != "" {
-		context.LoggerFromContext(ctx).Info("starting librato metrics reporter")
+		logger.Info("starting librato metrics reporter")
 		go librato.Librato(metrics.DefaultRegistry, time.Minute, cfg.LibratoEmail, cfg.LibratoToken, cfg.LibratoSource, []float64{0.95}, time.Millisecond)
 	} else {
-		context.LoggerFromContext(ctx).Info("starting logger metrics reporter")
+		logger.Info("starting logger metrics reporter")
 		go metrics.Log(metrics.DefaultRegistry, time.Minute, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
 	}
 
 	amqpConn, err := amqp.Dial(cfg.AmqpURI)
 	if err != nil {
-		context.LoggerFromContext(ctx).WithField("err", err).Error("couldn't connect to AMQP")
+		logger.WithField("err", err).Error("couldn't connect to AMQP")
 		return
 	}
 
@@ -96,50 +96,65 @@ func runWorker(c *cli.Context) {
 
 		err, ok := <-errChan
 		if ok {
-			context.LoggerFromContext(ctx).WithField("err", err).Error("amqp connection errored, terminating")
+			logger.WithField("err", err).Error("amqp connection errored, terminating")
 			cancel()
 		}
 	}()
 
-	context.LoggerFromContext(ctx).Debug("connected to AMQP")
+	logger.Debug("connected to AMQP")
 
 	generator := worker.NewBuildScriptGenerator(cfg)
+	logger.WithFields(logrus.Fields{
+		"build_script_generator": fmt.Sprintf("%#v", generator),
+	}).Debug("built")
+
 	provider, err := backend.NewProvider(cfg.ProviderName, cfg.ProviderConfig)
 	if err != nil {
-		context.LoggerFromContext(ctx).WithField("err", err).Error("couldn't create backend provider")
+		logger.WithField("err", err).Error("couldn't create backend provider")
 		return
 	}
-
-	context.LoggerFromContext(ctx).WithFields(logrus.Fields{
-		"provider": provider,
-	}).Debug("built provider")
+	logger.WithFields(logrus.Fields{
+		"provider": fmt.Sprintf("%#v", provider),
+	}).Debug("built")
 
 	commandDispatcher := worker.NewCommandDispatcher(ctx, amqpConn)
+	logger.WithFields(logrus.Fields{
+		"command_dispatcher": fmt.Sprintf("%#v", commandDispatcher),
+	}).Debug("built")
+
 	go commandDispatcher.Run()
 
 	pool := worker.NewProcessorPool(cfg.Hostname, ctx, cfg.HardTimeout, amqpConn,
 		provider, generator, commandDispatcher)
 
 	pool.SkipShutdownOnLogTimeout = cfg.SkipShutdownOnLogTimeout
+	logger.WithFields(logrus.Fields{
+		"pool": pool,
+	}).Debug("built")
 
 	signalChan := make(chan os.Signal, 1)
 	signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT)
 	go func() {
 		sig := <-signalChan
 		if sig == syscall.SIGINT {
-			context.LoggerFromContext(ctx).Info("SIGTERM received, starting graceful shutdown")
+			logger.Info("SIGTERM received, starting graceful shutdown")
 			pool.GracefulShutdown()
 		} else {
-			context.LoggerFromContext(ctx).Info("SIGINT received, shutting down immediately")
+			logger.Info("SIGINT received, shutting down immediately")
 			cancel()
 		}
 	}()
 
+	logger.WithFields(logrus.Fields{
+		"pool_size":  cfg.PoolSize,
+		"queue_name": cfg.QueueName,
+	}).Debug("running pool")
+
 	pool.Run(cfg.PoolSize, cfg.QueueName)
 
 	err = amqpConn.Close()
 	if err != nil {
-		context.LoggerFromContext(ctx).WithField("err", err).Error("couldn't close AMQP connection cleanly")
+		logger.WithField("err", err).Error("couldn't close AMQP connection cleanly")
 		return
 	}
 }

src/github.com/travis-ci/worker/config/config.go

@@ -3,7 +3,6 @@ package config
 import (
 	"fmt"
 	"io"
-	"os"
 	"sort"
 	"strings"
 	"time"
@@ -49,71 +48,36 @@ type Config struct {
 	BuildCacheS3SecretAccessKey string
 }
 
-func cfgString(c *cli.Context, name, dflt string) string {
-	for _, strval := range []string{c.String(name), c.GlobalString(name), dflt} {
-		if strval != "" {
-			return strval
-		}
-	}
-	return dflt
-}
-
-func cfgInt(c *cli.Context, name string, dflt int) int {
-	for _, intval := range []int{c.Int(name), c.GlobalInt(name), dflt} {
-		if intval != 0 {
-			return intval
-		}
-	}
-
-	return dflt
-}
-
-func cfgDuration(c *cli.Context, name string, dflt time.Duration) time.Duration {
-	for _, durval := range []time.Duration{c.Duration(name), c.GlobalDuration(name), dflt} {
-		if durval != zeroDuration {
-			return durval
-		}
-	}
-
-	return dflt
-}
-
-func cfgBool(c *cli.Context, name string, dflt bool) bool {
-	return c.GlobalBool(name) || c.Bool(name) || dflt
-}
-
 func ConfigFromCLIContext(c *cli.Context) *Config {
-	hostname, _ := os.Hostname()
-
 	cfg := &Config{
-		AmqpURI:       cfgString(c, "amqp-uri", defaultAmqpURI),
-		PoolSize:      cfgInt(c, "pool-size", defaultPoolSize),
-		BuildAPIURI:   cfgString(c, "build-api-uri", ""),
-		ProviderName:  cfgString(c, "provider-name", defaultProviderName),
-		QueueName:     cfgString(c, "queue-name", ""),
-		LibratoEmail:  cfgString(c, "librato-email", ""),
-		LibratoToken:  cfgString(c, "librato-token", ""),
-		LibratoSource: cfgString(c, "librato-source", ""),
-		SentryDSN:     cfgString(c, "sentry-dsn", ""),
-		Hostname:      cfgString(c, "hostname", hostname),
-		HardTimeout:   cfgDuration(c, "hard-timeout", defaultHardTimeout),
-
-		BuildAPIInsecureSkipVerify: cfgBool(c, "build-api-insecure-skip-verify", false),
-		SkipShutdownOnLogTimeout:   cfgBool(c, "skip-shutdown-on-log-timeout", false),
-
-		BuildCacheFetchTimeout:      cfgDuration(c, "build-cache-fetch-timeout", defaultBuildCacheFetchTimeout),
-		BuildCachePushTimeout:       cfgDuration(c, "build-cache-push-timeout", defaultBuildCachePushTimeout),
-		BuildAptCache:               cfgString(c, "build-apt-cache", ""),
-		BuildNpmCache:               cfgString(c, "build-npm-cache", ""),
-		BuildParanoid:               cfgBool(c, "build-paranoid", true),
-		BuildFixResolvConf:          cfgBool(c, "build-fix-resolv-conf", false),
-		BuildFixEtcHosts:            cfgBool(c, "build-fix-etc-hosts", false),
-		BuildCacheType:              cfgString(c, "build-cache-type", ""),
-		BuildCacheS3Scheme:          cfgString(c, "build-cache-s3-scheme", ""),
-		BuildCacheS3Region:          cfgString(c, "build-cache-s3-region", ""),
-		BuildCacheS3Bucket:          cfgString(c, "build-cache-s3-bucket", ""),
-		BuildCacheS3AccessKeyID:     cfgString(c, "build-cache-s3-access-key-id", ""),
-		BuildCacheS3SecretAccessKey: cfgString(c, "build-cache-s3-secret-access-key", ""),
+		AmqpURI:       c.String("amqp-uri"),
+		PoolSize:      c.Int("pool-size"),
+		BuildAPIURI:   c.String("build-api-uri"),
+		ProviderName:  c.String("provider-name"),
+		QueueName:     c.String("queue-name"),
+		LibratoEmail:  c.String("librato-email"),
+		LibratoToken:  c.String("librato-token"),
+		LibratoSource: c.String("librato-source"),
+		SentryDSN:     c.String("sentry-dsn"),
+		Hostname:      c.String("hostname"),
+		HardTimeout:   c.Duration("hard-timeout"),
+
+		BuildAPIInsecureSkipVerify: c.Bool("build-api-insecure-skip-verify"),
+		SkipShutdownOnLogTimeout:   c.Bool("skip-shutdown-on-log-timeout"),
+
+		BuildCacheFetchTimeout:      c.Duration("build-cache-fetch-timeout"),
+		BuildCachePushTimeout:       c.Duration("build-cache-push-timeout"),
+		BuildAptCache:               c.String("build-apt-cache"),
+		BuildNpmCache:               c.String("build-npm-cache"),
+		BuildParanoid:               c.Bool("build-paranoid"),
+		BuildFixResolvConf:          c.Bool("build-fix-resolv-conf"),
+		BuildFixEtcHosts:            c.Bool("build-fix-etc-hosts"),
+		BuildCacheType:              c.String("build-cache-type"),
+		BuildCacheS3Scheme:          c.String("build-cache-s3-scheme"),
+		BuildCacheS3Region:          c.String("build-cache-s3-region"),
+		BuildCacheS3Bucket:          c.String("build-cache-s3-bucket"),
+		BuildCacheS3AccessKeyID:     c.String("build-cache-s3-access-key-id"),
+		BuildCacheS3SecretAccessKey: c.String("build-cache-s3-secret-access-key"),
 	}
 
 	cfg.ProviderConfig = ProviderConfigFromEnviron(cfg.ProviderName)

src/github.com/travis-ci/worker/config/package.go → src/github.com/travis-ci/worker/config/defaults.go

@@ -1,6 +1,9 @@
 package config
 
-import "time"
+import (
+	"os"
+	"time"
+)
 
 var (
 	defaultAmqpURI                   = "amqp://"
@@ -9,4 +12,5 @@ var (
 	defaultHardTimeout, _            = time.ParseDuration("50m")
 	defaultBuildCacheFetchTimeout, _ = time.ParseDuration("5m")
 	defaultBuildCachePushTimeout, _  = time.ParseDuration("5m")
+	defaultHostname, _               = os.Hostname()
 )

src/github.com/travis-ci/worker/config/flags.go

@@ -11,8 +11,8 @@ var (
 	Flags = []cli.Flag{
 		cli.StringFlag{
 			Name:   "amqp-uri",
-			Usage:  "The URI to the AMQP server to connect to",
 			Value:  defaultAmqpURI,
+			Usage:  "The URI to the AMQP server to connect to",
 			EnvVar: twEnvVars("AMQP_URI"),
 		},
 		cli.IntFlag{
@@ -59,6 +59,7 @@ var (
 		},
 		cli.StringFlag{
 			Name:   "hostname",
+			Value:  defaultHostname,
 			Usage:  "Host name used in log output to identify the source of a job",
 			EnvVar: twEnvVars("HOSTNAME"),
 		},
@@ -70,8 +71,16 @@ var (
 		},
 
 		// build script generator flags
-		cli.DurationFlag{Name: "build-cache-fetch-timeout", EnvVar: twEnvVars("BUILD_CACHE_FETCH_TIMEOUT")},
-		cli.DurationFlag{Name: "build-cache-push-timeout", EnvVar: twEnvVars("BUILD_CACHE_PUSH_TIMEOUT")},
+		cli.DurationFlag{
+			Name:   "build-cache-fetch-timeout",
+			Value:  defaultBuildCacheFetchTimeout,
+			EnvVar: twEnvVars("BUILD_CACHE_FETCH_TIMEOUT"),
+		},
+		cli.DurationFlag{
+			Name:   "build-cache-push-timeout",
+			Value:  defaultBuildCachePushTimeout,
+			EnvVar: twEnvVars("BUILD_CACHE_PUSH_TIMEOUT"),
+		},
 		cli.StringFlag{Name: "build-apt-cache", EnvVar: twEnvVars("BUILD_APT_CACHE")},
 		cli.StringFlag{Name: "build-npm-cache", EnvVar: twEnvVars("BUILD_NPM_CACHE")},
 		cli.BoolFlag{Name: "build-paranoid", EnvVar: twEnvVars("BUILD_PARANOID")},
@@ -102,5 +111,12 @@ var (
 )
 
 func twEnvVars(key string) string {
-	return strings.ToUpper(fmt.Sprintf("TRAVIS_WORKER_%s,%s", key, key))
+	return strings.ToUpper(strings.Join(twEnvVarsSlice(key), ","))
+}
+
+func twEnvVarsSlice(key string) []string {
+	return []string{
+		fmt.Sprintf("TRAVIS_WORKER_%s", key),
+		key,
+	}
 }

utils/fmtpolice

@@ -15,12 +15,10 @@ gofmt_check () {
 }
 if [[ ! -n $1 ]]; then
   for f in $(git ls-files '*.go' | grep -v Deps) ; do
-    echo ${f}
     gofmt_check ${f}
   done
 else
   for f in $(find $1 -type f -iname *.go | grep -v Deps); do
-    echo ${f}
     gofmt_check ${f}
   done
 fi