Migrate Auditbeat, Heartbeat and Metricbeat to Python3

auditbeat/tests/system/auditbeat.py

@@ -24,7 +24,7 @@ def setUpClass(self):
 
     def create_file(self, path, contents):
         f = open(path, 'wb')
-        f.write(contents)
+        f.write(bytes(contents, "utf-8"))
         f.close()
 
     def check_event(self, event, expected):

auditbeat/tests/system/test_show_command.py

@@ -73,15 +73,15 @@ def test_show_auditd_rules(self):
         os.unlink(output_file)
         assert len(lines) >= len(rules)
         # get rid of automatic rule
-        if '-F key=rule' not in lines[0]:
+        if b'-F key=rule' not in lines[0]:
             del lines[0]
 
         for i in range(len(rules)):
             expected = rules[i]
             got = lines[i].strip()
-            assert expected == got, \
+            assert expected == got.decode("utf-8"), \
                 "rule {0} doesn't match. expected='{1}' got='{2}'".format(
-                    i, expected, got
+                    i, expected, got.decode("utf-8")
                 )
 
     @unittest.skipUnless(is_root(), "Requires root")
@@ -107,7 +107,7 @@ def test_show_auditd_status(self):
         self.run_beat(extra_args=['show', 'auditd-status'],
                       exit_code=0,
                       output=output_file)
-        fhandle = os.fdopen(fd, 'rb')
+        fhandle = os.fdopen(fd, 'r')
         lines = fhandle.readlines()
         fhandle.close()
         os.unlink(output_file)

heartbeat/tests/system/heartbeat.py

@@ -29,7 +29,7 @@ def do_GET(self):
                 if "write_delay" in kwargs:
                     sleep(float(kwargs["write_delay"]))
 
-                self.wfile.write(content)
+                self.wfile.write(bytes(content, "utf-8"))
 
         server = http.server.HTTPServer(('localhost', 0), HTTPHandler)
 

libbeat/tests/system/beat/compose.py

@@ -113,9 +113,9 @@ def _setup_advertised_host(cls, project, service):
         info.mode = 0o100644
         info.size = len(content)
 
-        data = io.StringIO()
+        data = io.BytesIO()
         tar = tarfile.TarFile(fileobj=data, mode='w')
-        tar.addfile(info, io.StringIO(content))
+        tar.addfile(info, fileobj=io.BytesIO(content.encode("utf-8")))
         tar.close()
 
         containers = project.containers(service_names=[service])
@@ -228,6 +228,6 @@ def service_log_contains(cls, service, msg):
         log = cls.get_service_log(service)
         counter = 0
         for line in log.splitlines():
-            if line.find(msg) >= 0:
+            if line.find(msg.encode("utf-8")) >= 0:
                 counter += 1
         return counter > 0

libbeat/tests/system/requirements.txt

@@ -4,11 +4,11 @@ cached-property==1.4.2
 certifi==2018.1.18
 chardet==3.0.4
 docker==3.7.0
-docker-compose==1.24.1
+docker-compose==1.25.0
 docker-pycreds==0.4.0
 dockerpty==0.4.1
 docopt==0.6.2
-elasticsearch==6.3.1
+elasticsearch==7.1.0
 enum34==1.1.6
 future==0.18.2
 idna==2.6

metricbeat/module/apache/test_apache.py

@@ -57,7 +57,7 @@ def test_output(self):
         # Waits until CPULoad is part of the status
         while not found:
             res = urllib.request.urlopen(hosts[0] + "/server-status?auto").read()
-            if "CPULoad" in res:
+            if b"CPULoad" in res:
                 found = True
             time.sleep(0.5)
 
@@ -76,15 +76,15 @@ def test_output(self):
         self.assert_fields_are_documented(evt)
 
     def verify_fields(self, evt):
-        self.assertItemsEqual(self.de_dot(APACHE_FIELDS), evt.keys())
+        self.assertCountEqual(self.de_dot(APACHE_FIELDS), evt.keys())
         apache_status = evt["apache"]["status"]
         if self.old_apache_version():
-            self.assertItemsEqual(
+            self.assertCountEqual(
                 self.de_dot(APACHE_OLD_STATUS_FIELDS), apache_status.keys())
         else:
-            self.assertItemsEqual(
+            self.assertCountEqual(
                 self.de_dot(APACHE_STATUS_FIELDS), apache_status.keys())
-            self.assertItemsEqual(
+            self.assertCountEqual(
                 self.de_dot(CPU_FIELDS), apache_status["cpu"].keys())
             # There are more fields that could be checked.
 

metricbeat/module/elasticsearch/test_elasticsearch.py

@@ -74,7 +74,6 @@ def test_metricsets(self, metricset):
         self.check_metricset("elasticsearch", metricset, self.get_hosts(), self.FIELDS +
                              ["service"], extras={"index_recovery.active_only": "false"})
 
-    @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
     def test_xpack(self):
         """
         elasticsearch-xpack module tests

metricbeat/module/mysql/test_mysql.py

@@ -38,7 +38,7 @@ def test_status(self):
         self.assertEqual(len(output), 1)
         evt = output[0]
 
-        self.assertItemsEqual(self.de_dot(MYSQL_FIELDS), evt.keys(), evt)
+        self.assertCountEqual(self.de_dot(MYSQL_FIELDS), evt.keys(), evt)
 
         status = evt["mysql"]["status"]
         assert status["connections"] > 0

metricbeat/tests/system/metricbeat.py

@@ -75,7 +75,7 @@ def assert_no_logged_warnings(self, replace=None):
             for r in replace:
                 pattern = self.build_log_regex(r)
                 log = pattern.sub("", log)
-        self.assertNotRegexpMatches(log, "\tERROR\t|\tWARN\t")
+        self.assertNotRegex(log, "\tERROR\t|\tWARN\t")
 
     def build_log_regex(self, message):
         return re.compile(r"^.*\t(?:ERROR|WARN)\t.*" + message + r".*$", re.MULTILINE)
@@ -104,7 +104,7 @@ def check_metricset(self, module, metricset, hosts, fields=[], extras=[]):
 
         fields = COMMON_FIELDS + fields
         print(fields)
-        self.assertItemsEqual(self.de_dot(fields), evt.keys())
+        self.assertCountEqual(self.de_dot(fields), evt.keys())
 
         self.assert_fields_are_documented(evt)
 

metricbeat/tests/system/requirements.txt

@@ -1,3 +1,3 @@
 kafka-python==1.4.3
-elasticsearch==6.3.1
+elasticsearch==7.1.0
 semver==2.8.1

metricbeat/tests/system/test_cmd.py

@@ -180,7 +180,7 @@ def touch(self, path):
         open(path, 'a').close()
 
     def write_system_yml(self):
-        with open(self.working_dir + "/modules.d/system.yml", "wb") as f:
+        with open(self.working_dir + "/modules.d/system.yml", "w") as f:
             f.write("""
 - module: system
   period: 10s
@@ -189,7 +189,7 @@ def write_system_yml(self):
     - memory""")
 
     def write_nginx_yml(self):
-        with open(self.working_dir + "/modules.d/nginx.yml", "wb") as f:
+        with open(self.working_dir + "/modules.d/nginx.yml", "w") as f:
             f.write("""
 - module: nginx
   period: 10s

metricbeat/tests/system/test_consul.py

@@ -43,14 +43,14 @@ def test_output(self):
         self.assertEqual(len(output), 1)
         evt = output[0]
 
-        self.assertItemsEqual(self.de_dot(CONSUL_FIELDS), evt.keys())
+        self.assertCountEqual(self.de_dot(CONSUL_FIELDS), evt.keys())
         consul_agent = evt["consul"]["agent"]
 
         consul_agent.pop("raft", None)
         consul_agent.pop("autopilot", None)
 
         print(consul_agent)
-        self.assertItemsEqual(self.de_dot(AGENT_FIELDS), consul_agent.keys())
+        self.assertCountEqual(self.de_dot(AGENT_FIELDS), consul_agent.keys())
 
         assert(consul_agent["runtime"]["heap_objects"] > 0)
 

metricbeat/tests/system/test_golang.py

@@ -27,7 +27,7 @@ def test_stats(self):
         output = self.read_output_json()
         evt = output[0]
 
-        self.assertItemsEqual(self.de_dot(GOLANG_FIELDS), evt.keys(), evt)
+        self.assertCountEqual(self.de_dot(GOLANG_FIELDS), evt.keys(), evt)
         assert evt["golang"]["heap"]["allocations"]["total"] > 0
 
         self.assert_fields_are_documented(evt)

metricbeat/tests/system/test_haproxy.py

@@ -20,7 +20,7 @@ def _test_info(self):
         self.assertEqual(len(output), 1)
         evt = output[0]
 
-        self.assertItemsEqual(self.de_dot(HAPROXY_FIELDS + ["process"]), evt.keys(), evt)
+        self.assertCountEqual(self.de_dot(HAPROXY_FIELDS + ["process"]), evt.keys(), evt)
 
         self.assert_fields_are_documented(evt)
 
@@ -48,7 +48,7 @@ def _test_stat(self):
 
         for evt in output:
             print(evt)
-            self.assertItemsEqual(self.de_dot(HAPROXY_FIELDS + ["process"]), evt.keys(), evt)
+            self.assertCountEqual(self.de_dot(HAPROXY_FIELDS + ["process"]), evt.keys(), evt)
             self.assert_fields_are_documented(evt)
 
     @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")

metricbeat/tests/system/test_http.py

@@ -37,7 +37,7 @@ def test_json(self):
 
         del evt["http"]["test"]["hello"]
 
-        self.assertItemsEqual(self.de_dot(HTTP_FIELDS), evt.keys(), evt)
+        self.assertCountEqual(self.de_dot(HTTP_FIELDS), evt.keys(), evt)
 
         self.assert_fields_are_documented(evt)
 
@@ -70,7 +70,7 @@ def test_server(self):
         # Delete dynamic namespace part for fields comparison
         del evt["http"]["server"]
 
-        self.assertItemsEqual(self.de_dot(HTTP_FIELDS), evt.keys(), evt)
+        self.assertCountEqual(self.de_dot(HTTP_FIELDS), evt.keys(), evt)
 
         self.assert_fields_are_documented(evt)
 

metricbeat/tests/system/test_kubernetes.py

@@ -71,7 +71,7 @@ def _test_metricset(self, metricset, expected_events, hosts):
         self.assertEqual(len(output), expected_events)
         evt = output[0]
 
-        self.assertItemsEqual(self.de_dot(KUBERNETES_FIELDS), evt.keys(), evt)
+        self.assertCountEqual(self.de_dot(KUBERNETES_FIELDS), evt.keys(), evt)
 
         self.assert_fields_are_documented(evt)
 

metricbeat/tests/system/test_mongodb.py

@@ -31,6 +31,6 @@ def test_status(self):
         self.assertEqual(len(output), 1)
         evt = output[0]
 
-        self.assertItemsEqual(self.de_dot(MONGODB_FIELDS + ["process"]), evt.keys())
+        self.assertCountEqual(self.de_dot(MONGODB_FIELDS + ["process"]), evt.keys())
 
         self.assert_fields_are_documented(evt)

metricbeat/tests/system/test_nats.py

@@ -30,7 +30,7 @@ def test_stats(self):
         self.assertEqual(len(output), 1)
         evt = output[0]
 
-        self.assertItemsEqual(self.de_dot(NATS_FIELDS), evt.keys(), evt)
+        self.assertCountEqual(self.de_dot(NATS_FIELDS), evt.keys(), evt)
 
         self.assert_fields_are_documented(evt)
 
@@ -55,7 +55,7 @@ def test_connections(self):
         self.assertEqual(len(output), 1)
         evt = output[0]
 
-        self.assertItemsEqual(self.de_dot(NATS_FIELDS), evt.keys(), evt)
+        self.assertCountEqual(self.de_dot(NATS_FIELDS), evt.keys(), evt)
 
         self.assert_fields_are_documented(evt)
 
@@ -80,7 +80,7 @@ def test_routes(self):
         self.assertEqual(len(output), 1)
         evt = output[0]
 
-        self.assertItemsEqual(self.de_dot(NATS_FIELDS), evt.keys(), evt)
+        self.assertCountEqual(self.de_dot(NATS_FIELDS), evt.keys(), evt)
 
         self.assert_fields_are_documented(evt)
 
@@ -105,7 +105,7 @@ def test_subscriptions(self):
         self.assertEqual(len(output), 1)
         evt = output[0]
 
-        self.assertItemsEqual(self.de_dot(NATS_FIELDS), evt.keys(), evt)
+        self.assertCountEqual(self.de_dot(NATS_FIELDS), evt.keys(), evt)
 
         self.assert_fields_are_documented(evt)
 

metricbeat/tests/system/test_phpfpm.py

@@ -29,6 +29,6 @@ def test_info(self):
         self.assertEqual(len(output), 1)
         evt = output[0]
 
-        self.assertItemsEqual(self.de_dot(PHPFPM_FIELDS), evt.keys(), evt)
+        self.assertCountEqual(self.de_dot(PHPFPM_FIELDS), evt.keys(), evt)
 
         self.assert_fields_are_documented(evt)

metricbeat/tests/system/test_postgresql.py

@@ -14,7 +14,7 @@ def common_checks(self, output):
 
         for evt in output:
             top_level_fields = metricbeat.COMMON_FIELDS + ["postgresql"]
-            self.assertItemsEqual(self.de_dot(top_level_fields), evt.keys())
+            self.assertCountEqual(self.de_dot(top_level_fields), evt.keys())
 
             self.assert_fields_are_documented(evt)
 

metricbeat/tests/system/test_processors.py

@@ -33,13 +33,13 @@ def test_drop_fields(self):
 
         print(evt)
         print(list(evt.keys()))
-        self.assertItemsEqual(self.de_dot([
+        self.assertCountEqual(self.de_dot([
             'agent', '@timestamp', 'system', 'metricset.module',
             'metricset.rtt', 'metricset.name', 'host', 'service', 'ecs', 'event'
         ]), evt.keys())
         cpu = evt["system"]["cpu"]
         print(list(cpu.keys()))
-        self.assertItemsEqual(self.de_dot([
+        self.assertCountEqual(self.de_dot([
             "system", "cores", "user", "softirq", "iowait",
             "idle", "irq", "steal", "nice", "total"
         ]), cpu.keys())

metricbeat/tests/system/test_prometheus.py

@@ -28,6 +28,6 @@ def test_stats(self):
         output = self.read_output_json()
         evt = output[0]
 
-        self.assertItemsEqual(self.de_dot(PROMETHEUS_FIELDS), evt.keys(), evt)
+        self.assertCountEqual(self.de_dot(PROMETHEUS_FIELDS), evt.keys(), evt)
 
         self.assert_fields_are_documented(evt)

metricbeat/tests/system/test_redis.py

@@ -45,11 +45,11 @@ def test_info(self):
         evt = output[0]
 
         fields = REDIS_FIELDS + ["process", "os"]
-        self.assertItemsEqual(self.de_dot(fields), evt.keys())
+        self.assertCountEqual(self.de_dot(fields), evt.keys())
         redis_info = evt["redis"]["info"]
-        self.assertItemsEqual(self.de_dot(REDIS_INFO_FIELDS), redis_info.keys())
-        self.assertItemsEqual(self.de_dot(CLIENTS_FIELDS), redis_info["clients"].keys())
-        self.assertItemsEqual(self.de_dot(CPU_FIELDS), redis_info["cpu"].keys())
+        self.assertCountEqual(self.de_dot(REDIS_INFO_FIELDS), redis_info.keys())
+        self.assertCountEqual(self.de_dot(CLIENTS_FIELDS), redis_info["clients"].keys())
+        self.assertCountEqual(self.de_dot(CPU_FIELDS), redis_info["cpu"].keys())
         self.assert_fields_are_documented(evt)
 
     @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
@@ -83,9 +83,9 @@ def test_keyspace(self):
         self.assertEqual(len(output), 1)
         evt = output[0]
 
-        self.assertItemsEqual(self.de_dot(REDIS_FIELDS), evt.keys())
+        self.assertCountEqual(self.de_dot(REDIS_FIELDS), evt.keys())
         redis_info = evt["redis"]["keyspace"]
-        self.assertItemsEqual(self.de_dot(REDIS_KEYSPACE_FIELDS), redis_info.keys())
+        self.assertCountEqual(self.de_dot(REDIS_KEYSPACE_FIELDS), redis_info.keys())
         self.assert_fields_are_documented(evt)
 
     @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
@@ -123,7 +123,7 @@ def test_key(self):
         self.assertEqual(len(output), 1)
         evt = output[0]
 
-        self.assertItemsEqual(self.de_dot(REDIS_FIELDS), evt.keys())
+        self.assertCountEqual(self.de_dot(REDIS_FIELDS), evt.keys())
         self.assert_fields_are_documented(evt)
 
     @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
@@ -153,12 +153,12 @@ def test_module_processors(self):
         self.assertEqual(len(output), 1)
         evt = output[0]
 
-        self.assertItemsEqual(self.de_dot(REDIS_FIELDS), evt.keys())
+        self.assertCountEqual(self.de_dot(REDIS_FIELDS), evt.keys())
         redis_info = evt["redis"]["info"]
         print(redis_info)
-        self.assertItemsEqual(fields, redis_info.keys())
-        self.assertItemsEqual(self.de_dot(CLIENTS_FIELDS), redis_info["clients"].keys())
-        self.assertItemsEqual(self.de_dot(CPU_FIELDS), redis_info["cpu"].keys())
+        self.assertCountEqual(fields, redis_info.keys())
+        self.assertCountEqual(self.de_dot(CLIENTS_FIELDS), redis_info["clients"].keys())
+        self.assertCountEqual(self.de_dot(CPU_FIELDS), redis_info["cpu"].keys())
 
 
 class TestRedis4(Test):