Tag image names in metricbeat docker compose files (#13775)

Tag image names so they can be more easily reused, cached and shared.

Modify all Dockerfiles related to metricbeat resources to parameterize
their versions.

Build args are not used anymore in docker compose, and environment is
used instead. The reason is that build args couldn't be used to tag
the images. Build args are still passed to Dockerfiles, and they don't
have default values now, they must be set in the docker compose file.

Some services that are being tested with multiple versions have been
refactorized to be parameterized so they can share most of their
Dockerfiles and docker compose definitions.

Testing framework doesn't force rebuild of images now. If an image
exists with the same tag this image will be used. Before trying to start
a service, it tries to pull its image to avoid building it if it is available.
Both pull and build will reuse local layers if they can.

Part of #13471 and #12909.

libbeat/tests/compose/compose.go

@@ -65,7 +65,6 @@ func EnsureUp(t testing.TB, service string, options ...UpOption) HostInfo {
 	upOptions := UpOptions{
 		Timeout: 60 * time.Second,
 		Create: CreateOptions{
-			Build:         true,
 			ForceRecreate: true,
 		},
 	}

libbeat/tests/compose/wrapper.go

@@ -183,6 +183,15 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 		args = append(args, service)
 	}
 
+	// Try to pull the image before building it
+	var stderr bytes.Buffer
+	pull := d.cmd(ctx, "pull", "--ignore-pull-failures", service)
+	pull.Stdout = nil
+	pull.Stderr = &stderr
+	if err := pull.Run(); err != nil {
+		return errors.Wrapf(err, "failed to pull images using docker-compose: %s", stderr.String())
+	}
+
 	err := d.cmd(ctx, "up", args...).Run()
 	if err != nil {
 		return err

libbeat/tests/system/beat/compose.py

@@ -10,6 +10,7 @@
 
 if INTEGRATION_TESTS:
     from compose.cli.command import get_project
+    from compose.config.environment import Environment
     from compose.service import BuildAction
     from compose.service import ConvergenceStrategy
 
@@ -22,8 +23,8 @@ class ComposeMixin(object):
     # List of required services to run INTEGRATION_TESTS
     COMPOSE_SERVICES = []
 
-    # Additional build arguments for docker compose build
-    COMPOSE_BUILD_ARGS = {}
+    # Additional environment variables for docker compose
+    COMPOSE_ENV = {}
 
     # timeout waiting for health (seconds)
     COMPOSE_TIMEOUT = 300
@@ -51,13 +52,12 @@ def is_healthy(container):
             return container.inspect()['State']['Health']['Status'] == 'healthy'
 
         project = cls.compose_project()
-        project.build(
-            service_names=cls.COMPOSE_SERVICES,
-            build_args=cls.COMPOSE_BUILD_ARGS)
+        project.pull(
+            ignore_pull_failures=True,
+            service_names=cls.COMPOSE_SERVICES)
         project.up(
             strategy=ConvergenceStrategy.always,
             service_names=cls.COMPOSE_SERVICES,
-            do_build=BuildAction.skip,
             timeout=30)
 
         # Wait for them to be healthy
@@ -196,11 +196,15 @@ def compose_project_name(cls):
         def positivehash(x):
             return hash(x) % ((sys.maxsize+1) * 2)
 
-        return "%s_%X" % (basename, positivehash(frozenset(cls.COMPOSE_BUILD_ARGS.items())))
+        return "%s_%X" % (basename, positivehash(frozenset(cls.COMPOSE_ENV.items())))
 
     @classmethod
     def compose_project(cls):
-        return get_project(cls.find_compose_path(), project_name=cls.compose_project_name())
+        env = Environment(os.environ.copy())
+        env.update(cls.COMPOSE_ENV)
+        return get_project(cls.find_compose_path(),
+                           project_name=cls.compose_project_name(),
+                           environment=env)
 
     @classmethod
     def find_compose_path(cls):