Merge pull request #25 from wpoely86/bytes

Properly convert bytes to real bytes, using 1024 basis (AB#18742)
vub-hpc · Dec 12, 2024 · 66af4e2 · 66af4e2
2 parents 645bf56 + 1f0ef21
commit 66af4e2
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 18 deletions.
diff --git a/tests/data/pixiu b/tests/data/pixiu
@@ -167,15 +167,15 @@ data = [
             "alarm_severity": "Major",
             "alarm_object": "OceanStor-Distributed-Storage",
             "alarm_id": "0xFEA6A000B",
-            "used_space": 1024000000,
+            "used_space": 1073741824,
             "quota_used_type": "space",
             "quota_type": "directory",
             "dtree": "test1",
             "dtree_id": 4101,
             "namespace": "admin",
             "namespace_id": 71,
             "quota_limit_type": "hard",
-            "quota_space_limit": 1024000000,
+            "quota_space_limit": 1073741824,
             "path": "/",
         },
     },
@@ -187,7 +187,7 @@ data = [
             "alarm_severity": "Warning",
             "alarm_object": "OceanStor-Distributed-Storage",
             "alarm_id": "0xFEA6A000F",
-            "used_space": 8191000000,
+            "used_space": 8588886016,
             "quota_used_type": "space",
             "quota_type": "user",
             "username": "vsc10042",
@@ -196,7 +196,7 @@ data = [
             "namespace": "user",
             "namespace_id": 70,
             "quota_limit_type": "soft",
-            "quota_space_limit": 6144000000,
+            "quota_space_limit": 6442450944,
             "path": "/100",
         },
     },
@@ -208,7 +208,7 @@ data = [
             "alarm_severity": "Warning",
             "alarm_object": "OceanStor-Distributed-Storage",
             "alarm_id": "0xFEA6A0011",
-            "used_space": 8192000000,
+            "used_space": 8589934592,
             "quota_used_type": "space",
             "quota_type": "user",
             "username": "vsc10042",
@@ -217,7 +217,7 @@ data = [
             "namespace": "user",
             "namespace_id": 70,
             "quota_limit_type": "hard",
-            "quota_space_limit": 8192000000,
+            "quota_space_limit": 8589934592,
             "path": "/100",
         },
     },
@@ -269,11 +269,11 @@ data = [
             "namespace_id": 242,
             "dtree": "--",
             "dtree_id": 0,
-            "used_space": 18000000,
+            "used_space": 18874368,
             "quota_used_type": "space",
             "quota_type": "directory",
             "quota_limit_type": "soft",
-            "quota_space_limit": 15000000,
+            "quota_space_limit": 15728640,
             "path": "/",
         },
     },
@@ -286,11 +286,11 @@ data = [
             "namespace_id": 182,
             "dtree": "--",
             "dtree_id": 0,
-            "used_space": 8388608000000,
+            "used_space": 8796093022208,
             "quota_used_type": "space",
             "quota_type": "directory",
             "quota_limit_type": "hard",
-            "quota_space_limit": 8388608000000,
+            "quota_space_limit": 8796093022208,
             "path": "/",
         },
     },
@@ -340,8 +340,8 @@ data = [
             "namespace_id": 70,
             "quota_limit_type": "soft",
             "path": "/110",
-            "quota_space_limit": 5851000000,
-            "used_space": 6108000000,
+            "quota_space_limit": 6135218176,
+            "used_space": 6404702208,
         },
     },
     {
@@ -427,11 +427,11 @@ data = [
             "namespace_id": 592,
             "dtree": "--",
             "dtree_id": 0,
-            "used_space": 10000,
+            "used_space": 10240,
             "quota_used_type": "space",
             "quota_type": "directory",
             "quota_limit_type": "hard",
-            "quota_space_limit": 10000,
+            "quota_space_limit": 10240,
             "path": "/",
         },
     },

diff --git a/tests/logstash_7.6.2.conf b/tests/logstash_7.6.2.conf
@@ -37,10 +37,16 @@ filter {
         }
     }
 
-    if ("_grokparsefailure" not in [tags]) {
-        mutate {
-            remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp", "KEY_EQ_VALUEDATA" ]
-        }
+    if "_grokparsefailure" not in [tags] {
+      mutate {
+        # in the future remove the field 'message' too
+        remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp", "KEY_EQ_VALUEDATA", "int" ]
+        convert => { "success" => "boolean" }
+
+        # we need MB converted to MiB for bytes2human
+        gsub => [ "used_space", "([0-9]+[kKmMgGtTpP])[bB]", "\1ib",
+                  "quota_space_limit", "([0-9]+[kKmMgGtTpP])[bB]", "\1ib" ]
+      }
     }
 
     if [jube_id] {