Merge branch 'develop' into rpm-dpkg-search-filename

.github/workflows/unit-testing.yaml

@@ -18,24 +18,24 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Clone uac repo
+      - name: Checkout uac repository
         uses: actions/checkout@v4
         with:
           repository: tclahr/uac
           path: uac
 
-      - name: Clone ushunit repo
+      - name: Checkout ushunit repository
         uses: actions/checkout@v4
         with:
           repository: tclahr/ushunit
           ref: main
           path: ushunit
 
-      - name: Clone uac-tests repo
+      - name: Checkout uac-tests repository
         uses: actions/checkout@v4
         with:
           repository: tclahr/uac-tests
-          ref: main
+          ref: ${{ github.event.pull_request.base.ref }}
           path: uac-tests
 
       - name: Run tests

CHANGELOG.md

@@ -2,87 +2,3 @@
 
 ## DEVELOPMENT VERSION
 
-### Features
-
-- UAC now completely skips an artifact file (YAML) that has no artifacts to be collected for the target operating system. You can use '--artifacts list [OPERATING_SYSTEM]' to display artifacts for a specific operating system only.
-- New output file formats:
-  - none: Collected data will not be archived or compressed. Instead, it will be copied directly to an output directory ([#188](https://github.com/tclahr/uac/issues/188)).
-  - zip: Collected data will be archived and compressed into a zip file. Additionally, you can create a password-protected zip file using the '--output-password' option ([#149](https://github.com/tclahr/uac/issues/149)).
-- You can now set a custom output file name using the '-o/--output-base-name' command line option. Variables are available to format the filename ([#179](https://github.com/tclahr/uac/issues/179)).
-- Now you have the option to supply a file path to a custom profile located outside the profiles directory.
-- Now you have the option to supply a file path to a custom artifact located outside the artifacts directory ([#154](https://github.com/tclahr/uac/issues/154)).
-- Now you can have the option to supply a file path to a custom config file located outside the config directory using the '-c/--config' command line option.
-- New remote transfer options for Amazon, Google and IBM cloud storage locations.
-- UAC will now use 'wget' to transfer files to remote cloud storage locations when 'curl' is not available.
-- You can now increase the verbosity level using the '-v/--verbose' command line option. Enabling a higher verbosity level will result in the display of all executed commands.
-- UAC will now use the built-in function 'astrings' to extract strings from binary files when 'strings' is not available on the system.
-- The message 'The strings command requires the command line developer tools.' will no longer appear on macOS systems without developer tools installed ([#171](https://github.com/tclahr/uac/issues/171)).
-- Error messages generated by executed commands (stderr) are now recorded in the uac.log file ([#150](https://github.com/tclahr/uac/issues/150)).
-- New '-H/--hash-collected' command line option. Enabling this option will cause UAC to hash all collected files and save the results in a hash file. To accomplish this, all collected data must first be copied to the destination directory. Therefore, ensure you have twice the free space available on the system: once for the collected data and once for the output file. Additionally, note that this process will increase the running time ([#189](https://github.com/tclahr/uac/issues/189)).
-- New '-t/--max-thread' command line option. It can be used to specify the number of files that will be processed in parallel by the 'hash' and 'stat' collectors.
-- You can now validate profiles using the '--validate-profile' command line option.
-
-### Artifacts
-
-- bodyfile/bodyfile.yaml: Updated to remove max_depth limit.
-- files/applications/whatsapp.yaml: Added collection of WhatsApp Desktop files [macos].
-- files/logs/additional_logs.yaml: Artifact was renamed to advanced_log_search.yaml.
-- files/logs/relink.yaml: Added collection of the kernel relink log file [openbsd] [Herbert-Karl](https://github.com/Herbert-Karl)).
-- files/system/acct.yaml: Added collection of system accounting files [freebsd, netbsd, openbsd] [Herbert-Karl](https://github.com/Herbert-Karl)).
-- files/system/dev_db.yaml: Added collection of the database file used for device lookups [netbsd, openbsd] [Herbert-Karl](https://github.com/Herbert-Karl)).
-- files/system/dev_shm.yaml: Updated to increase max_file_size to 10MB.
-- files/system/locate_db.yaml: Added collection of the database file used by locate command, representing a snapshot of the virtual file system accessible with minimal permissions [freebsd, netbsd, openbsd] [Herbert-Karl](https://github.com/Herbert-Karl)).
-- files/system/netscaler.yaml: Updated to increase max_file_size to 10MB.
-- files/system/run_shm.yaml: Updated to increase max_file_size to 10MB.
-- files/system/security_backups.yaml: Added collection of file backups and hashes created by the integrated security script [freebsd, netbsd, openbsd] [Herbert-Karl](https://github.com/Herbert-Karl)).
-- files/system/tmp.yaml: Updated to increase max_file_size to 10MB.
-- files/system/var_tmp.yaml: Updated to increase max_file_size to 10MB.
-- hash_executables/hash_executables.yaml: Updated to remove max_depth and max_file_size properties.
-- live_response/containers/jls.yaml: Added collection of jails used on FreeBSD systems [freebsd] [Herbert-Karl](https://github.com/Herbert-Karl)).
-- live_response/hardware/dmesg.yaml: Updated collection of console message bufffer [esxi, freebsd, netscaler, openbsd, solaris] [Herbert-Karl](https://github.com/Herbert-Karl)).
-- live_response/process/deleted.yaml: Collection of deleted processes will no longer use dd conv=swab. The binary file will be collected in its raw format now [linux].
-- live_response/process/deleted.yaml: Updated to fix the collection of open files of (malicious) processes [linux] [mnrkbys](https://github.com/mnrkbys)).
-- live_response/process/hash_running_processes.yaml: Updated to add support to hash running processes on FreeBSD systems that are using procfs (/proc) [freebsd].
-- live_response/process/procfs_information.yaml: Added artifact collection using cat when strings is not available.
-- live_response/process/strings_running_processes.yaml: Added collection of strings from running processes for ESXi systems [esxi].
-- live_response/process/strings_running_processes.yaml: Added condition to check whether developer tools are installed before running strings on macOS [macos].
-- live_response/process/strings_running_processes.yaml: Added support for collecting strings even when the strings command is unavailable. In such cases, the built-in astrings command will be used instead [all].
-- live_response/system/hidden_directories.yaml: Updated to remove max_depth limit.
-- live_response/system/hidden_files.yaml: Updated to remove max_depth limit.
-- live_response/system/lastcomm.yaml: Added collection of the last commands executed in a reverse order based on the default and historic accounting file [freebsd, netbsd, openbsd] [Herbert-Karl](https://github.com/Herbert-Karl)).
-- live_response/system/sgid.yaml: Updated to remove max_depth limit.
-- live_response/system/socket_files.yaml: Updated to remove max_depth limit.
-- live_response/system/suid.yaml: Updated to remove max_depth limit.
-- live_response/system/world_writable_directories.yaml: Updated to remove max_depth limit.
-- live_response/system/world_writable_files.yaml: Updated to remove max_depth limit.
-- live_response/system/zoneadm.yaml: Artifact was moved to live_response/containers directory [Herbert-Karl](https://github.com/Herbert-Karl)).
-
-### Command Line Option Changes
-
-- '--date-range-start' was renamed to '--start-date' ([#186](https://github.com/tclahr/uac/issues/186)).
-- '--date-range-end' was renamed to '--end-date' ([#186](https://github.com/tclahr/uac/issues/186)).
-- '--validate-artifacts-file' was renamed to '--validate-artifact'.
-- '--s3-presigned-url' was renamed to '--aws-s3-presigned-url'.
-- '--s3-presigned-url-log-file' was renamed to '--aws-s3-presigned-url-log-file'.
-- '--ibm-cos-url', '--ibm-cos-url-log-file' and '--ibm-cloud-api-key' were removed and now transfers to IBM cloud should be done using '--s3-provider', '--s3-region', '--s3-bucket' and '--s3-token' options.
-
-### Artifacts Properties
-
-- The 'output_directory' property is now mandatory for the following collectors: command, find, hash and stat.
-- Introduced a new 'condition' property that ensures the collection runs only if the specified condition returns true.
-
-### uac.conf
-
-- Introduced a new global 'max_depth' configuration option to limit the depth of directory tree searches globally.
-
-### Tools
-
-- Statically linked 'zip' is now available for the following systems:
-  - linux/esxi (arm, arm64, i386 and x86_64)
-  - freebsd/netscaler (i386 and x86_64)
-- 'avml' and 'linux_procmemdump.sh' tools were moved to the 'bin' directory.
-- AVML updated to v0.14.0.
-
-### Deprecated
-
-- Android support was removed, but UAC can still be executed on Android systems using '--operating-system linux' option.

README.md

@@ -41,7 +41,7 @@ UAC is a Live Response collection script for Incident Response that makes use of
 
 UAC reads YAML files on the fly and, based on their contents, collects relevant artifacts. This makes UAC very customizable and extensible.
 
-[![uac_collection](https://tclahr.github.io/uac-docs/img/uac_collection.gif)](#)
+[![uac_collection](https://tclahr.github.io/uac-docs/img/uac_3_collection.gif)](#)
 
 ## 📘 Documentation
 

artifacts/files/applications/git.yaml

@@ -0,0 +1,28 @@
+version: 1.0
+artifacts:
+# Git hooks/Git pager can be used to run persistence.
+# ref: https://www.elastic.co/security-labs/sequel-on-persistence-mechanisms
+  -
+    description: Collect Git hooks under .git/hooks directory.
+    supported_os: [linux, macos]
+    collector: file
+    path: /
+    path_pattern: ["*/.git/hooks/*"]
+    file_type: [f]
+  -
+    description: Collect /etc/gitconfig file.
+    supported_os: [linux, macos]
+    collector: file
+    path: /etc/gitconfig
+  -
+    description: Collect ~/.gitconfig file.
+    supported_os: [linux, macos]
+    collector: file
+    path: /%user_home%/.gitconfig
+    exclude_nologin_users: true
+  -
+    description: Collect ~/.config/git/gitconfig file.
+    supported_os: [linux, macos]
+    collector: file
+    path: /%user_home%/.config/git/config
+    exclude_nologin_users: true

artifacts/files/applications/lesshst.yaml

@@ -0,0 +1,26 @@
+version: 1.0
+artifacts:
+# https://wiki.archlinux.org/title/XDG_Base_Directory
+# https://github.com/gwsw/less/issues/153
+# https://www.greenwoodsoftware.com/less/news.590.html
+# https://www.greenwoodsoftware.com/less/news.600.html
+  -
+    description: Collect less history file. This file is used to store search string.
+    supported_os: [aix, freebsd, linux, macos, netbsd, netscaler, openbsd, solaris]
+    collector: file
+    path: /%user_home%/.lesshst
+    exclude_nologin_users: true
+  -
+    description: Collect less history file. This file is used to store search string.
+    supported_os: [aix, freebsd, linux, macos, netbsd, netscaler, openbsd, solaris]
+    collector: file
+    # $XDG_STATE_HOME/lesshst
+    path: /%user_home%/.local/state/lesshst
+    exclude_nologin_users: true
+  -
+    description: Collect less history file. This file is used to store search string.
+    supported_os: [aix, freebsd, linux, macos, netbsd, netscaler, openbsd, solaris]
+    collector: file
+    # $XDG_DATA_HOME/lesshst
+    path: /%user_home%/.local/share/lesshst
+    exclude_nologin_users: true

artifacts/files/logs/apache.yaml

@@ -1,27 +1,27 @@
-version: 1.0
+version: 2.0
 artifacts:
   -
     description: Collect Apache logs.
-    supported_os: [aix, freebsd, linux, macos, netbsd, netscaler, openbsd, solaris]
+    supported_os: [all]
     collector: file
     path: /var/log
     name_pattern: ["access_log*", "access.log*", "error_log*", "error.log*"]
     max_file_size: 1073741824 # 1GB
   -
     description: Collect Apache logs.
-    supported_os: [aix, freebsd, linux, macos, netbsd, netscaler, openbsd, solaris]
+    supported_os: [all]
     collector: file
     path: /var/log/apache
     max_file_size: 1073741824 # 1GB
   -
     description: Collect Apache logs.
-    supported_os: [aix, freebsd, linux, macos, netbsd, netscaler, openbsd, solaris]
+    supported_os: [all]
     collector: file
     path: /var/log/apache2
     max_file_size: 1073741824 # 1GB
   -
     description: Collect Apache logs.
-    supported_os: [aix, freebsd, linux, macos, netbsd, netscaler, openbsd, solaris]
+    supported_os: [all]
     collector: file
     path: /var/log/httpd
     max_file_size: 1073741824 # 1GB

artifacts/files/logs/journal.yaml

@@ -0,0 +1,8 @@
+version: 1.0
+artifacts:
+  -
+    description: Collect journal log files.
+    supported_os: [linux]
+    collector: file
+    path: /
+    name_pattern: ["*.journal"]

artifacts/files/logs/nginx.yaml

@@ -2,14 +2,14 @@ version: 1.0
 artifacts:
   -
     description: Collect nginx logs.
-    supported_os: [aix, freebsd, linux, macos, netbsd, netscaler, openbsd, solaris]
+    supported_os: [all]
     collector: file
     path: /var/log
     name_pattern: ["*access_log*", "*access.log*", "*error_log*", "*error.log*"]
     max_file_size: 1073741824 # 1GB
   -
     description: Collect nginx logs.
-    supported_os: [aix, freebsd, linux, macos, netbsd, netscaler, openbsd, solaris]
+    supported_os: [all]
     collector: file
     path: /var/log/nginx
     max_file_size: 1073741824 # 1GB

artifacts/files/logs/run_log.yaml

@@ -0,0 +1,9 @@
+version: 1.0
+artifacts:
+  -
+    description: Collect /run/log files.
+    supported_os: [linux]
+    collector: file
+    path: /run/log
+    max_file_size: 1073741824 # 1GB
+

artifacts/files/logs/tomcat.yaml

@@ -1,8 +1,8 @@
-version: 1.0
+version: 2.0
 artifacts:
   -
     description: Collect Apache Tomcat logs.
-    supported_os: [aix, freebsd, linux, macos, netbsd, netscaler, openbsd, solaris]
+    supported_os: [all]
     collector: file
     path: /
     name_pattern: ["access_log*", "error_log*", "httpd-access.log*", "httpd-error.log*", "catalina.out"]

artifacts/files/packages/apt.yaml

@@ -0,0 +1,9 @@
+version: 1.0
+# APT can be used to run persistence.
+# ref: https://www.elastic.co/security-labs/sequel-on-persistence-mechanisms
+artifacts:
+  -
+    description: Collect script files under /etc/apt/apt.conf.d/ directory.
+    supported_os: [linux]
+    collector: file
+    path: /etc/apt/apt.conf.d

artifacts/files/packages/dnf.yaml

@@ -0,0 +1,16 @@
+version: 1.0
+# DNF can be used to run persistence.
+# ref: https://www.elastic.co/security-labs/sequel-on-persistence-mechanisms
+artifacts:
+  -
+    description: Collect configuration files under /etc/dnf/pluginconf.d/ directory.
+    supported_os: [linux]
+    collector: file
+    path: /etc/dnf/pluginconf.d
+  -
+    description: Collect script files under dnf-plugins directories.
+    supported_os: [linux]
+    collector: file
+    path: /
+    name_pattern: ["dnf-plugins"]
+    file_type: [d]

artifacts/files/packages/pkg_contents.yaml

@@ -1,5 +1,10 @@
-version: 2.0
+version: 2.1
 artifacts:
+  -
+    description: Collect installed packages database.
+    supported_os: [freebsd]
+    collector: file
+    path: /var/db/pkg/local.sqlite
   -
     description: Collect package table of contents files.
     supported_os: [netbsd, openbsd]
@@ -17,3 +22,4 @@ artifacts:
     supported_os: [solaris]
     collector: file
     path: /var/pkg/publisher/*/pkg
+

artifacts/files/packages/yum.yaml

@@ -0,0 +1,14 @@
+version: 1.0
+# YUM can be used to run persistence.
+# ref: https://www.elastic.co/security-labs/sequel-on-persistence-mechanisms
+artifacts:
+  -
+    description: Collect configuration files under /etc/yum/pluginconf.d/ directory.
+    supported_os: [linux]
+    collector: file
+    path: /etc/yum/pluginconf.d
+  -
+    description: Collect script files under /usr/lib/yum-plugins/ directory.
+    supported_os: [linux]
+    collector: file
+    path: /usr/lib/yum-plugins

artifacts/files/system/acct.yaml

@@ -1,7 +1,7 @@
-version: 1.0
+version: 2.0
 artifacts:
   # system accounting files, covering processes that terminated on the system, allowing one to see past program executions
-  # this is deactivated by default, but quite usefull when active
+  # this is deactivated by default, but quite useful when active
   -
     description: Collect system accounting files.
     supported_os: [freebsd, netbsd, openbsd]
@@ -19,4 +19,24 @@ artifacts:
     supported_os: [freebsd, netbsd, openbsd]
     collector: file
     path: /var/account/savacct
+    ignore_date_range: true
+  -
+    description: Collect system accounting files.
+    supported_os: [solaris]
+    collector: file
+    path: /var/adm/pacct*
+    ignore_date_range: true
+  -
+    description: Collect system accounting summary files.
+    supported_os: [solaris]
+    collector: file
+    path: /var/adm/acct
+    max_file_size: 1073741824 # 1GB
+    ignore_date_range: true
+  -
+    description: Collect extended system accounting files from default location.
+    supported_os: [solaris]
+    collector: file
+    path: /var/adm/exacct
+    max_file_size: 1073741824 # 1GB
     ignore_date_range: true