kola: update tests to use ss instead of netstat

Update kola tests to use ss instead of the deprecated netstat,
and adjust tests to expect ss output.

This should fix coreos/fedora-coreos-tracker#552.

The following tests have been updated:

fcos.network.listeners
crio.network
podman.network-multi (currently disabled)

mantle/kola/tests/crio/crio.go

@@ -306,7 +306,7 @@ func crioNetwork(c cluster.TestCluster) {
 	talker := func(ctx context.Context) error {
 		// Wait until listener is ready before trying anything
 		for {
-			_, err := c.SSH(dest, "sudo netstat -tulpn|grep 9988")
+			_, err := c.SSH(dest, "sudo ss -tulpn|grep 9988")
 			if err == nil {
 				break // socket is ready
 			}

mantle/kola/tests/misc/network.go

@@ -59,21 +59,20 @@ type listener struct {
 func checkListeners(c cluster.TestCluster, expectedListeners []listener) error {
 	m := c.Machines()[0]
 
-	output := c.MustSSH(m, "sudo netstat -plutn")
+	output := c.MustSSH(m, "sudo ss -plutn")
 
 	processes := strings.Split(string(output), "\n")
 	// verify header is as expected
 	if len(processes) < 2 {
-		c.Fatalf("expected at least two lines of nestat output: %q", output)
+		c.Fatalf("expected at least two lines of ss output: %q", output)
 	}
-	if processes[0] != "Active Internet connections (only servers)" {
-		c.Fatalf("netstat output has changed format: %q", output)
-	}
-	if !regexp.MustCompile(`Proto\s+Recv-Q\s+Send-Q\s+Local Address\s+Foreign Address\s+State\s+PID/Program name`).MatchString(processes[1]) {
-		c.Fatalf("netstat output has changed format: %q", output)
+	// ss output's header sometimes does not have whitespace between "Peer Address:Port" and "Process"
+	headerRegex := `Netid\s+State\s+Recv-Q\s+Send-Q\s+Local Address:Port\s+Peer Address:Port\s*Process`
+	if !regexp.MustCompile(headerRegex).MatchString(processes[0]) {
+		c.Fatalf("ss output has changed format: %q", processes[0])
 	}
 	// skip header
-	processes = processes[2:]
+	processes = processes[1:]
 
 NextProcess:
 	for _, line := range processes {
@@ -84,14 +83,16 @@ NextProcess:
 			c.Fatalf("unexpected number of parts on line: %q in output %q", line, output)
 		}
 		proto := parts[0]
-		portdata := strings.Split(parts[3], ":")
-		port := portdata[len(portdata)-1]
-		pidProgramParts := strings.SplitN(parts[len(parts)-1], "/", 2)
-		if len(pidProgramParts) != 2 {
-			c.Errorf("%v did not contain pid and program parts; full output: %q", parts[6], output)
+		portData := strings.Split(parts[4], ":")
+		port := portData[len(portData)-1]
+		processData := parts[len(parts)-1]
+		pidStr := regexp.MustCompile(`pid=\d+`).FindString(processData)
+		processStr := regexp.MustCompile(`".+"`).FindString(processData)
+		if pidStr == "" || processStr == "" {
+			c.Errorf("%v did not contain pid and program; full output: %q", processData, output)
 			continue
 		}
-		pid, process := pidProgramParts[0], pidProgramParts[1]
+		pid, process := pidStr[4:], processStr[1:len(processStr)-1]
 
 		for _, expected := range expectedListeners {
 			if strings.HasPrefix(proto, expected.protocol) && // allow expected tcp to match tcp6
@@ -107,7 +108,7 @@ NextProcess:
 			continue
 		}
 
-		c.Logf("full netstat output: %q", output)
+		c.Logf("full ss output: %q", output)
 		return fmt.Errorf("Unexpected listener process: %q", line)
 	}
 

mantle/kola/tests/podman/podman.go

@@ -312,7 +312,7 @@ func podmanNetworkTest(c cluster.TestCluster) {
 	talker := func(ctx context.Context) error {
 		// Wait until listener is ready before trying anything
 		for {
-			_, err := c.SSH(dest, "sudo netstat -tulpn | grep 9988")
+			_, err := c.SSH(dest, "sudo ss -tulpn | grep 9988")
 			if err == nil {
 				break // socket is ready
 			}