[system] Adds tags.yml file so they appear under the Security Solution UI and upgrades package spec to version 3.0.0

packages/system/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.45.0"
+  changes:
+    - description: Upgrade to package spec 3.0.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8206
 - version: "1.44.0"
   changes:
     - description: Enable TSDB by default for process datastream. This improves storage usage and query performance. For more details, see https://www.elastic.co/guide/en/elasticsearch/reference/current/tsds.html

packages/system/data_stream/auth/_dev/test/pipeline/test-auth-ubuntu1204.log-config.yml

@@ -1,5 +1,6 @@
 fields:
-  event.timezone: "+0000"
+  event:
+    timezone: "+0000"
 dynamic_fields:
-  event.ingested: "^.*$"
+  "event.ingested": "^.*$"
   "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}Z$"

packages/system/data_stream/auth/_dev/test/pipeline/test-auth-ubuntu1204.log-expected.json

@@ -62,7 +62,7 @@
             "system": {
                 "auth": {
                     "sudo": {
-                        "command": "/bin/sh -c echo BECOME-SUCCESS-lhspyyxxlfzpytwsebjoegenjxyjombo; LANG=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 /usr/bin/python /home/vagrant/.ansible/tmp/ansible-tmp-1486675177.72-26828938879074/get_url; rm -rf /home/vagrant/.ansible/tmp/ansible-tmp-1486675177.72-26828938879074/ \u003e/dev/null 2\u003e\u00261",
+                        "command": "/bin/sh -c echo BECOME-SUCCESS-lhspyyxxlfzpytwsebjoegenjxyjombo; LANG=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 /usr/bin/python /home/vagrant/.ansible/tmp/ansible-tmp-1486675177.72-26828938879074/get_url; rm -rf /home/vagrant/.ansible/tmp/ansible-tmp-1486675177.72-26828938879074/ >/dev/null 2>&1",
                         "pwd": "/home/vagrant",
                         "tty": "pts/0",
                         "user": "root"
@@ -181,7 +181,7 @@
             "system": {
                 "auth": {
                     "sudo": {
-                        "command": "/bin/sh -c echo BECOME-SUCCESS-lwzhcvorajmjyxsrqydafzapoeescwaf; rc=flag; [ -r /etc/metricbeat/metricbeat.yml ] || rc=2; [ -f /etc/metricbeat/metricbeat.yml ] || rc=1; [ -d /etc/metricbeat/metricbeat.yml ] \u0026\u0026 rc=3; python -V 2\u003e/dev/null || rc=4; [ x\"$rc\" != \"xflag\" ] \u0026\u0026 echo \"${rc}  \"/etc/metricbeat/metricbeat.yml \u0026\u0026 exit 0; (python -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();#012afile = open(\"'/etc/metricbeat/metricbeat.yml'\", \"rb\")#012buf = afile.read(BLOCKSIZE)#012while len(buf) \u003e 0:#012#011hasher.update(buf)#012#011buf = afile.read(BLOCKSIZE)#012afile.close()#012print(hasher.hexdigest())' 2\u003e/dev/null) || (python -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();#012afile = open(\"'/etc/metricbeat/metricbeat.yml'\", \"rb\")#012buf = afile.read(BLOCKSIZE)#012while len(buf) \u003e 0:#012#011hasher.update(buf)#012#011buf = afile.read(BLOCKSIZE)#012afile.close()#012print(hasher.hexdigest())' 2\u003e/dev/null) || (echo '0",
+                        "command": "/bin/sh -c echo BECOME-SUCCESS-lwzhcvorajmjyxsrqydafzapoeescwaf; rc=flag; [ -r /etc/metricbeat/metricbeat.yml ] || rc=2; [ -f /etc/metricbeat/metricbeat.yml ] || rc=1; [ -d /etc/metricbeat/metricbeat.yml ] && rc=3; python -V 2>/dev/null || rc=4; [ x\"$rc\" != \"xflag\" ] && echo \"${rc}  \"/etc/metricbeat/metricbeat.yml && exit 0; (python -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();#012afile = open(\"'/etc/metricbeat/metricbeat.yml'\", \"rb\")#012buf = afile.read(BLOCKSIZE)#012while len(buf) > 0:#012#011hasher.update(buf)#012#011buf = afile.read(BLOCKSIZE)#012afile.close()#012print(hasher.hexdigest())' 2>/dev/null) || (python -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();#012afile = open(\"'/etc/metricbeat/metricbeat.yml'\", \"rb\")#012buf = afile.read(BLOCKSIZE)#012while len(buf) > 0:#012#011hasher.update(buf)#012#011buf = afile.read(BLOCKSIZE)#012afile.close()#012print(hasher.hexdigest())' 2>/dev/null) || (echo '0",
                         "pwd": "/home/vagrant",
                         "tty": "pts/0",
                         "user": "root"
@@ -375,7 +375,7 @@
             "system": {
                 "auth": {
                     "sudo": {
-                        "command": "/bin/sh -c echo BECOME-SUCCESS-ippzqmywwjlstxlqlpyxbnzzgeigarma; rc=flag; [ -r /etc/heartbeat/heartbeat.yml ] || rc=2; [ -f /etc/heartbeat/heartbeat.yml ] || rc=1; [ -d /etc/heartbeat/heartbeat.yml ] \u0026\u0026 rc=3; python -V 2\u003e/dev/null || rc=4; [ x\"$rc\" != \"xflag\" ] \u0026\u0026 echo \"${rc}  \"/etc/heartbeat/heartbeat.yml \u0026\u0026 exit 0; (python -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();#012afile = open(\"'/etc/heartbeat/heartbeat.yml'\", \"rb\")#012buf = afile.read(BLOCKSIZE)#012while len(buf) \u003e 0:#012#011hasher.update(buf)#012#011buf = afile.read(BLOCKSIZE)#012afile.close()#012print(hasher.hexdigest())' 2\u003e/dev/null) || (python -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();#012afile = open(\"'/etc/heartbeat/heartbeat.yml'\", \"rb\")#012buf = afile.read(BLOCKSIZE)#012while len(buf) \u003e 0:#012#011hasher.update(buf)#012#011buf = afile.read(BLOCKSIZE)#012afile.close()#012print(hasher.hexdigest())' 2\u003e/dev/null) || (echo '0",
+                        "command": "/bin/sh -c echo BECOME-SUCCESS-ippzqmywwjlstxlqlpyxbnzzgeigarma; rc=flag; [ -r /etc/heartbeat/heartbeat.yml ] || rc=2; [ -f /etc/heartbeat/heartbeat.yml ] || rc=1; [ -d /etc/heartbeat/heartbeat.yml ] && rc=3; python -V 2>/dev/null || rc=4; [ x\"$rc\" != \"xflag\" ] && echo \"${rc}  \"/etc/heartbeat/heartbeat.yml && exit 0; (python -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();#012afile = open(\"'/etc/heartbeat/heartbeat.yml'\", \"rb\")#012buf = afile.read(BLOCKSIZE)#012while len(buf) > 0:#012#011hasher.update(buf)#012#011buf = afile.read(BLOCKSIZE)#012afile.close()#012print(hasher.hexdigest())' 2>/dev/null) || (python -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();#012afile = open(\"'/etc/heartbeat/heartbeat.yml'\", \"rb\")#012buf = afile.read(BLOCKSIZE)#012while len(buf) > 0:#012#011hasher.update(buf)#012#011buf = afile.read(BLOCKSIZE)#012afile.close()#012print(hasher.hexdigest())' 2>/dev/null) || (echo '0",
                         "pwd": "/home/vagrant",
                         "tty": "pts/0",
                         "user": "root"

packages/system/data_stream/auth/_dev/test/pipeline/test-auth.log-config.yml

@@ -1,5 +1,6 @@
 fields:
-  event.timezone: "+0000"
+  event:
+    timezone: "+0000"
 dynamic_fields:
-  event.ingested: "^.*$"
+  "event.ingested": "^.*$"
   "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}Z$"

packages/system/data_stream/auth/_dev/test/pipeline/test-multiline.log-config.yml

@@ -1,7 +1,8 @@
 dynamic_fields:
   "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}"
 fields:
-  event.timezone: "+0000"
+  event:
+    timezone: "+0000"
 multiline:
   # Pattern to match what is configured in log.yml.hbs.
   first_line_pattern: '^[^\s]'

packages/system/data_stream/auth/_dev/test/pipeline/test-secure-rhel7.log-config.yml

@@ -1,5 +1,6 @@
 fields:
-  event.timezone: "+0000"
+  event:
+    timezone: "+0000"
 dynamic_fields:
-  event.ingested: "^.*$"
+  "event.ingested": "^.*$"
   "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}Z$"

packages/system/data_stream/auth/_dev/test/pipeline/test-secure-rhel7.log-expected.json

@@ -82,7 +82,7 @@
             "host": {
                 "hostname": "slave22"
             },
-            "message": "pam_succeed_if(sshd:auth): requirement \"uid \u003e= 1000\" not met by user \"root\"",
+            "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"",
             "process": {
                 "name": "sshd",
                 "pid": 2738
@@ -170,7 +170,7 @@
             "host": {
                 "hostname": "slave22"
             },
-            "message": "PAM service(sshd) ignoring max retries; 5 \u003e 3",
+            "message": "PAM service(sshd) ignoring max retries; 5 > 3",
             "process": {
                 "name": "sshd",
                 "pid": 2738

packages/system/data_stream/auth/_dev/test/pipeline/test-timestamp.log-config.yml

@@ -1,4 +1,5 @@
 fields:
-  event.timezone: "+0000"
+  event:
+    timezone: "+0000"
 dynamic_fields:
-  event.ingested: ".*"
+  "event.ingested": ".*"

packages/system/data_stream/process/fields/fields.yml

@@ -11,7 +11,7 @@
         The full command-line used to start the process, including the arguments separated by space.
       ignore_above: 2048
     - name: env
-      type: object
+      type: flattened
       description: |
         The environment variables used to start the process. The data is available on FreeBSD, Linux, and OS X.
     - name: cpu

packages/system/data_stream/security/_dev/test/pipeline/test-4663.json-expected.json

@@ -31,7 +31,7 @@
                 },
                 "level": "information"
             },
-            "message": "\u003cEvent xmlns=\"http://schemas.microsoft.com/win/2004/08/events/event\"\u003e\u003cSystem\u003e \u003cProvider Name=\"Microsoft-Windows-Security-Auditing\" Guid=\"{54849625-5478-4994-A5BA-3E3B0328C30D}\" /\u003e\u003cEventID\u003e4663\u003c/EventID\u003e\u003cVersion\u003e1\u003c/Version\u003e\u003cLevel\u003e0\u003c/Level\u003e\u003cTask\u003e12800\u003c/Task\u003e\u003cOpcode\u003e0\u003c/Opcode\u003e\u003cKeywords\u003e0x8020000000000000\u003c/Keywords\u003e\u003cTimeCreated SystemTime=\"2015-09-18T22:13:54.770429700Z\" /\u003e\u003cEventRecordID\u003e273866\u003c/EventRecordID\u003e\u003cCorrelation /\u003e\u003cExecution ProcessID=\"516\" ThreadID=\"524\" /\u003e\u003cChannel\u003eSecurity\u003c/Channel\u003e\u003cComputer\u003eDC01.contoso.local\u003c/Computer\u003e\u003cSecurity /\u003e\u003c/System\u003e\u003cEventData\u003e\u003cData Name=\"SubjectUserSid\"\u003eS-1-5-21-3457937927-2839227994-823803824-1104\u003c/Data\u003e\u003cData Name=\"SubjectUserName\"\u003edadmin\u003c/Data\u003e\u003cData Name=\"SubjectDomainName\"\u003eCONTOSO\u003c/Data\u003e\u003cData Name=\"SubjectLogonId\"\u003e0x4367b\u003c/Data\u003e\u003cData Name=\"ObjectServer\"\u003eSecurity\u003c/Data\u003e\u003cData Name=\"ObjectType\"\u003eFile\u003c/Data\u003e\u003cData Name=\"ObjectName\"\u003eC:\\\\Documents\\\\HBI Data.txt\u003c/Data\u003e\u003cData Name=\"HandleId\"\u003e0x1bc\u003c/Data\u003e\u003cData Name=\"AccessList\"\u003e%%4417 %%4418\u003c/Data\u003e\u003cData Name=\"AccessMask\"\u003e0x6\u003c/Data\u003e\u003cData Name=\"ProcessId\"\u003e0x458\u003c/Data\u003e\u003cData Name=\"ProcessName\"\u003eC:\\\\Windows\\\\System32\\\\notepad.exe\u003c/Data\u003e\u003cData Name=\"ResourceAttributes\"\u003eS:AI(RA;ID;;;;WD;(\"Impact\\_MS\",TI,0x10020,3000))\u003c/Data\u003e\u003c/EventData\u003e\u003c/Event\u003e",
+            "message": "<Event xmlns=\"http://schemas.microsoft.com/win/2004/08/events/event\"><System> <Provider Name=\"Microsoft-Windows-Security-Auditing\" Guid=\"{54849625-5478-4994-A5BA-3E3B0328C30D}\" /><EventID>4663</EventID><Version>1</Version><Level>0</Level><Task>12800</Task><Opcode>0</Opcode><Keywords>0x8020000000000000</Keywords><TimeCreated SystemTime=\"2015-09-18T22:13:54.770429700Z\" /><EventRecordID>273866</EventRecordID><Correlation /><Execution ProcessID=\"516\" ThreadID=\"524\" /><Channel>Security</Channel><Computer>DC01.contoso.local</Computer><Security /></System><EventData><Data Name=\"SubjectUserSid\">S-1-5-21-3457937927-2839227994-823803824-1104</Data><Data Name=\"SubjectUserName\">dadmin</Data><Data Name=\"SubjectDomainName\">CONTOSO</Data><Data Name=\"SubjectLogonId\">0x4367b</Data><Data Name=\"ObjectServer\">Security</Data><Data Name=\"ObjectType\">File</Data><Data Name=\"ObjectName\">C:\\\\Documents\\\\HBI Data.txt</Data><Data Name=\"HandleId\">0x1bc</Data><Data Name=\"AccessList\">%%4417 %%4418</Data><Data Name=\"AccessMask\">0x6</Data><Data Name=\"ProcessId\">0x458</Data><Data Name=\"ProcessName\">C:\\\\Windows\\\\System32\\\\notepad.exe</Data><Data Name=\"ResourceAttributes\">S:AI(RA;ID;;;;WD;(\"Impact\\_MS\",TI,0x10020,3000))</Data></EventData></Event>",
             "winlog": {
                 "channel": "Security",
                 "computer_name": "DC01.contoso.local",

packages/system/data_stream/security/_dev/test/pipeline/test-4674.json-expected.json

@@ -38,7 +38,7 @@
                 },
                 "level": "information"
             },
-            "message": "\u003cEvent xmlns=\"http://schemas.microsoft.com/win/2004/08/events/event\"\u003e\u003cSystem\u003e\u003cProvider Name=\"Microsoft-Windows-Security-Auditing\" Guid=\"{54849625-5478-4994-A5BA-3E3B0328C30D}\" /\u003e\u003cEventID\u003e4674\u003c/EventID\u003e\u003cVersion\u003e0\u003c/Version\u003e\u003cLevel\u003e0\u003c/Level\u003e\u003cTask\u003e13056\u003c/Task\u003e\u003cOpcode\u003e0\u003c/Opcode\u003e\u003cKeywords\u003e0x8010000000000000\u003c/Keywords\u003e\u003cTimeCreated SystemTime=\"2015-10-09T00:22:36.237816000Z\" /\u003e\u003cEventRecordID\u003e1099680\u003c/EventRecordID\u003e\u003cCorrelation /\u003e\u003cExecution ProcessID=\"496\" ThreadID=\"504\" /\u003e\u003cChannel\u003eSecurity\u003c/Channel\u003e\u003cComputer\u003eDC01.contoso.local\u003c/Computer\u003e\u003cSecurity /\u003e\u003c/System\u003e\u003cEventData\u003e\u003cData Name=\"SubjectUserSid\"\u003eS-1-5-19\u003c/Data\u003e\u003cData Name=\"SubjectUserName\"\u003eLOCAL SERVICE\u003c/Data\u003e\u003cData Name=\"SubjectDomainName\"\u003eNT AUTHORITY\u003c/Data\u003e\u003cData Name=\"SubjectLogonId\"\u003e0x3e5\u003c/Data\u003e\u003cData Name=\"ObjectServer\"\u003eLSA\u003c/Data\u003e\u003cData Name=\"ObjectType\"\u003e-\u003c/Data\u003e\u003cData Name=\"ObjectName\"\u003e-\u003c/Data\u003e\u003cData Name=\"HandleId\"\u003e0x0\u003c/Data\u003e\u003cData Name=\"AccessMask\"\u003e16777216\u003c/Data\u003e\u003cData Name=\"PrivilegeList\"\u003eSeSecurityPrivilege\u003c/Data\u003e\u003cData Name=\"ProcessId\"\u003e0x1f0\u003c/Data\u003e\u003cData Name=\"ProcessName\"\u003eC:\\\\Windows\\\\System32\\\\lsass.exe\u003c/Data\u003e\u003c/EventData\u003e\u003c/Event\u003e",
+            "message": "<Event xmlns=\"http://schemas.microsoft.com/win/2004/08/events/event\"><System><Provider Name=\"Microsoft-Windows-Security-Auditing\" Guid=\"{54849625-5478-4994-A5BA-3E3B0328C30D}\" /><EventID>4674</EventID><Version>0</Version><Level>0</Level><Task>13056</Task><Opcode>0</Opcode><Keywords>0x8010000000000000</Keywords><TimeCreated SystemTime=\"2015-10-09T00:22:36.237816000Z\" /><EventRecordID>1099680</EventRecordID><Correlation /><Execution ProcessID=\"496\" ThreadID=\"504\" /><Channel>Security</Channel><Computer>DC01.contoso.local</Computer><Security /></System><EventData><Data Name=\"SubjectUserSid\">S-1-5-19</Data><Data Name=\"SubjectUserName\">LOCAL SERVICE</Data><Data Name=\"SubjectDomainName\">NT AUTHORITY</Data><Data Name=\"SubjectLogonId\">0x3e5</Data><Data Name=\"ObjectServer\">LSA</Data><Data Name=\"ObjectType\">-</Data><Data Name=\"ObjectName\">-</Data><Data Name=\"HandleId\">0x0</Data><Data Name=\"AccessMask\">16777216</Data><Data Name=\"PrivilegeList\">SeSecurityPrivilege</Data><Data Name=\"ProcessId\">0x1f0</Data><Data Name=\"ProcessName\">C:\\\\Windows\\\\System32\\\\lsass.exe</Data></EventData></Event>",
             "process": {
                 "executable": "C:\\\\Windows\\\\System32\\\\lsass.exe",
                 "name": "lsass.exe",

packages/system/data_stream/security/_dev/test/pipeline/test-common-config.yml

@@ -1,2 +1,2 @@
 dynamic_fields:
-  event.ingested: ".*"
+  "event.ingested": ".*"

packages/system/data_stream/syslog/_dev/test/pipeline/test-darwin-syslog-sample.log-config.yml

@@ -3,5 +3,6 @@ dynamic_fields:
 multiline:
   first_line_pattern: "^\\w+ \\d+ "
 fields:
-  event.kind: "event"
-  event.timezone: "GMT-0200"
+  event:
+    kind: "event"
+    timezone: "GMT-0200"

packages/system/data_stream/syslog/_dev/test/pipeline/test-darwin-syslog-sample.log-expected.json

@@ -12,7 +12,7 @@
             "host": {
                 "hostname": "a-mac-with-esc-key"
             },
-            "message": "2016-12-13 11:35:28.420 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSAgentApp updateProductWithProductID:usingEngine:] Checking for updates for \"All Products\" using engine \u003cKSUpdateEngine:0x100341a00\n\t\tticketStore=\u003cKSPersistentTicketStore:0x100204520 store=\u003cKSKeyedPersistentStore:0x100213290\n\t\t\tpath=\"/Users/tsg/Library/Google/GoogleSoftwareUpdate/TicketStore/Keystone.ticketstore\"\n\t\t\tlockFile=\u003cKSLockFile:0x1002160e0\n\t\t\t\tpath=\"/Users/tsg/Library/Google/GoogleSoftwareUpdate/TicketStore/Keystone.ticketstore.lock\"\n\t\t\t\tlocked=NO\n\t\t\t\u003e\n\t\t\u003e\u003e\n\t\tprocessor=\u003cKSActionProcessor:0x1003bb5f0\n\t\t\tdelegate=\u003cKSUpdateEngine:0x100341a00\u003e\n\t\t\tisProcessing=NO actionsCompleted=0 progress=0.00\n\t\t\terrors=0 currentActionErrors=0\n\t\t\tevents=0 currentActionEvents=0\n\t\t\tactionQueue=( )\n\t\t\u003e\n\t\tdelegate=(null)\n\t\tserverInfoStore=(null)\n\t\terrors=0\n\t\u003e",
+            "message": "2016-12-13 11:35:28.420 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSAgentApp updateProductWithProductID:usingEngine:] Checking for updates for \"All Products\" using engine <KSUpdateEngine:0x100341a00\n\t\tticketStore=<KSPersistentTicketStore:0x100204520 store=<KSKeyedPersistentStore:0x100213290\n\t\t\tpath=\"/Users/tsg/Library/Google/GoogleSoftwareUpdate/TicketStore/Keystone.ticketstore\"\n\t\t\tlockFile=<KSLockFile:0x1002160e0\n\t\t\t\tpath=\"/Users/tsg/Library/Google/GoogleSoftwareUpdate/TicketStore/Keystone.ticketstore.lock\"\n\t\t\t\tlocked=NO\n\t\t\t>\n\t\t>>\n\t\tprocessor=<KSActionProcessor:0x1003bb5f0\n\t\t\tdelegate=<KSUpdateEngine:0x100341a00>\n\t\t\tisProcessing=NO actionsCompleted=0 progress=0.00\n\t\t\terrors=0 currentActionErrors=0\n\t\t\tevents=0 currentActionEvents=0\n\t\t\tactionQueue=( )\n\t\t>\n\t\tdelegate=(null)\n\t\tserverInfoStore=(null)\n\t\terrors=0\n\t>",
             "process": {
                 "name": "GoogleSoftwareUpdateAgent",
                 "pid": 21412

packages/system/data_stream/syslog/_dev/test/pipeline/test-darwin-syslog.log-config.yml

@@ -3,4 +3,5 @@ dynamic_fields:
 multiline:
   first_line_pattern: "^Dec 13 "
 fields:
-  event.timezone: "GMT-0200"
+  event:
+    timezone: "GMT-0200"