Fix integration tests

- switched to configuration layout mandated by v2
- added `mage test` target that will run both unit and integration
  tests
- switched CI runs from just running unit tests to running all tests
- Skipped integration tests that require docker to run logstash or
  kafka, support for doing that is finalized in magefile.
- updated example config file
- add support for running under Windows with named pipes

Closes elastic#244

.ci/Jenkinsfile

@@ -91,7 +91,7 @@ pipeline {
               withGithubNotify(context: "Test-${PLATFORM}") {
                 dir("${BASE_DIR}"){
                   withMageEnv(){
-                    cmd(label: 'Go unitTest', script: 'mage unitTest')
+                    cmd(label: 'Go unitTest', script: 'mage test')
                   }
                 }
               }

NOTICE.txt

@@ -9,6 +9,38 @@ Third party libraries used by the Elastic Agent Shipper:
 ================================================================================
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/Microsoft/go-winio
+Version: v0.5.2
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/!microsoft/go-winio@v0.5.2/LICENSE:
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Microsoft
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/sarama
 Version: v1.19.1-0.20220310193331-ebc2b0d8eef3
@@ -6676,38 +6708,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
 
---------------------------------------------------------------------------------
-Dependency : github.com/Microsoft/go-winio
-Version: v0.5.2
-Licence type (autodetected): MIT
---------------------------------------------------------------------------------
-
-Contents of probable licence file $GOMODCACHE/github.com/!microsoft/go-winio@v0.5.2/LICENSE:
-
-The MIT License (MIT)
-
-Copyright (c) 2015 Microsoft
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-
-
 --------------------------------------------------------------------------------
 Dependency : github.com/OneOfOne/xxhash
 Version: v1.2.2

controller/controller_client_test.go

@@ -1,7 +1,7 @@
 // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
-//go:build !integration
+//go:build !integration && !windows
 
 package controller
 
@@ -88,7 +88,6 @@ func TestRemoveOutput(t *testing.T) {
 				for _, unit := range observed.Units {
 					t.Logf("current state for %s is: %s", unit.Id, unit.Message)
 				}
-
 			}
 		}
 		return nil
@@ -100,7 +99,6 @@ func TestRemoveOutput(t *testing.T) {
 	assert.True(t, restartedOutput, "gRPC input has stopped")
 	assert.True(t, outputHealthy, "input restarted")
 	assert.True(t, testStopped, "test has shut down")
-
 }
 
 func TestStopAllInputs(t *testing.T) {
@@ -190,7 +188,6 @@ func TestStopAllInputs(t *testing.T) {
 				for _, unit := range observed.Units {
 					t.Logf("current state for %s is: %s", unit.Id, unit.Message)
 				}
-
 			}
 		}
 		return nil
@@ -203,7 +200,6 @@ func TestStopAllInputs(t *testing.T) {
 	assert.True(t, newInput, "input restarted")
 	assert.True(t, testStopped, "test has shut down")
 	assert.True(t, inputRemoved, "input unit removed")
-
 }
 
 func TestChangeOutputType(t *testing.T) {
@@ -219,7 +215,8 @@ func TestChangeOutputType(t *testing.T) {
 			"type":    "elasticsearch",
 			"enabled": "true",
 			"hosts":   []interface{}{"localhost:9200"},
-		})}
+		}),
+	}
 
 	doneWaiter := &sync.WaitGroup{}
 	srvFunc := func(observed *proto.CheckinObserved) *proto.CheckinExpected {
@@ -284,7 +281,6 @@ func TestChangeOutputType(t *testing.T) {
 				for _, unit := range observed.Units {
 					t.Logf("current state for %s is: %s", unit.Id, unit.Message)
 				}
-
 			}
 		}
 		return nil
@@ -393,7 +389,6 @@ func TestAddingInputs(t *testing.T) {
 				for _, unit := range observed.Units {
 					t.Logf("current state for %s is: %s", unit.Id, unit.Message)
 				}
-
 			}
 		}
 		return nil
@@ -407,7 +402,6 @@ func TestAddingInputs(t *testing.T) {
 	assert.True(t, stoppedFirst, "stopped first unit")
 	assert.True(t, gotStopped, "units stopped")
 	assert.False(t, gotRestarted, "units restarted, they should not")
-
 }
 
 func TestBasicAgentControl(t *testing.T) {
@@ -446,7 +440,7 @@ func TestBasicAgentControl(t *testing.T) {
 			} else if unitsAreState(t, proto.State_HEALTHY, observed.Units) {
 				t.Logf("Got unit state healthy, sending STOPPED")
 				gotHealthy = true
-				//shutdown
+				// shutdown
 				unitIn.ConfigStateIdx++
 				unitIn.State = proto.State_STOPPED
 
@@ -469,7 +463,6 @@ func TestBasicAgentControl(t *testing.T) {
 				for _, unit := range observed.Units {
 					t.Logf("current state for %s is: %s", unit.Id, unit.Message)
 				}
-
 			}
 		}
 		return nil
@@ -480,7 +473,6 @@ func TestBasicAgentControl(t *testing.T) {
 	assert.True(t, gotConfig, "config state")
 	assert.True(t, gotHealthy, "healthy state")
 	assert.True(t, gotStopped, "stopped state")
-
 }
 
 func TestUnitLogChange(t *testing.T) {
@@ -522,7 +514,7 @@ func TestUnitLogChange(t *testing.T) {
 			} else if unitsAreState(t, proto.State_HEALTHY, observed.Units) && !gotHealthy {
 				t.Logf("Got unit state healthy, changing log level")
 				gotHealthy = true
-				//shutdown
+				// shutdown
 				unitOut.ConfigStateIdx++
 				unitOut.LogLevel = proto.UnitLogLevel_DEBUG
 				return &proto.CheckinExpected{
@@ -540,7 +532,7 @@ func TestUnitLogChange(t *testing.T) {
 			} else if unitsAreState(t, proto.State_HEALTHY, observed.Units) && gotHealthy {
 				// shut down
 				t.Logf("Got unit state healthy, stopping")
-				//check to see if log level has changed
+				// check to see if log level has changed
 				observedLogLevel = logp.GetLevel()
 				unitOut.ConfigStateIdx++
 				unitOut.State = proto.State_STOPPED
@@ -560,7 +552,6 @@ func TestUnitLogChange(t *testing.T) {
 				for _, unit := range observed.Units {
 					t.Logf("current state for %s is: %s", unit.Id, unit.Message)
 				}
-
 			}
 		}
 		return nil
@@ -573,7 +564,6 @@ func TestUnitLogChange(t *testing.T) {
 	assert.True(t, gotStopped, "stopped state")
 	assert.False(t, gotRestarted, "units restarted")
 	assert.Equal(t, zapcore.DebugLevel, observedLogLevel, "not expected log level")
-
 }
 
 func runServerTest(t *testing.T, implFunc mock.StubServerCheckinV2, waitUntil *sync.WaitGroup, token string) {
@@ -582,7 +572,6 @@ func runServerTest(t *testing.T, implFunc mock.StubServerCheckinV2, waitUntil *s
 	srv := mock.StubServerV2{
 		CheckinV2Impl: implFunc,
 		ActionImpl: func(response *proto.ActionResponse) error {
-
 			return nil
 		},
 		ActionsChan: make(chan *mock.PerformAction, 100),
@@ -605,7 +594,6 @@ func runServerTest(t *testing.T, implFunc mock.StubServerCheckinV2, waitUntil *s
 
 	err := runController(ctx, validClient)
 	assert.NoError(t, err)
-
 }
 
 func createClient(token string, port int) client.V2 {

controller/run_test.go

@@ -6,31 +6,26 @@ package controller
 
 import (
 	"context"
-	"net"
-	"os"
-	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-shipper/config"
+	"github.com/elastic/elastic-agent-shipper/grpcserver"
 	"github.com/elastic/elastic-agent-shipper/output"
 )
 
 func TestUnmanaged(t *testing.T) {
 	_ = logp.DevelopmentSetup()
-	serverAddr := filepath.Join(os.TempDir(), "test-unmanaged-shipper.sock")
+	serverAddr := grpcserver.TestSocketName(t.TempDir())
 	cfg := config.DefaultConfig()
 	cfg.Type = "console"
 	cfg.Shipper.Output.Console = &output.ConsoleConfig{Enabled: true}
 	cfg.Shipper.Server.Server = serverAddr
 
-	defer func() {
-		_ = os.Remove(serverAddr)
-	}()
-
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 
 	go func() {
@@ -41,25 +36,20 @@ func TestUnmanaged(t *testing.T) {
 	for {
 		select {
 		case <-ctx.Done():
-			t.Fatalf("timed out waiting for unix socket")
+			t.Fatalf("timed out trying to dial %s", serverAddr)
 		default:
 		}
-		_, err := os.Stat(serverAddr)
-		if !os.IsNotExist(err) {
-			break
+		// _, err := os.Stat(serverAddr)
+		// if err != nil && errors.Is(err, fs.ErrNotExist) {
+		// 	continue
+		// }
+		con, err := grpcserver.TestDial(serverAddr)
+		if err != nil && (strings.Contains(err.Error(), "connection refused") || strings.Contains(err.Error(), "no such file")) {
+			continue
 		}
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = con.Close() })
+		break
 	}
-	// remove the file now that we know it's there
-	defer func() {
-		_ = os.Remove(serverAddr)
-	}()
-
-	// basic test, make sure output is running
-	con, err := net.Dial("unix", serverAddr)
-	require.NoError(t, err)
-	defer func() {
-		_ = con.Close()
-	}()
-
 	cancel()
 }