Merge pull request #287 from tclahr/release/3.0.0

Release/3.0.0

.github/workflows/shellcheck.yaml

@@ -1,42 +1,27 @@
 name: ShellCheck
 
 on:
-  push:
-    branches:
-      - develop
-      - main
-      - 'releases/**'
-    paths:
-      - 'lib/**'
-      - 'tools/**'
-      - 'uac'
-
   pull_request:
     branches:
       - develop
       - main
-      - 'releases/**'
     paths:
       - 'lib/**'
-      - 'tools/**'
       - 'uac'
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
 jobs:
-  shellcheck:
+  build:
     name: ShellCheck
     runs-on: ubuntu-latest
 
     steps:
       - name: Clone uac repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: uac
 
       - name: Run ShellCheck
         uses: ludeeus/action-shellcheck@master
-        with:
-          ignore_paths: artifacts bin config profiles
-

.github/workflows/unit-testing.yaml

@@ -0,0 +1,44 @@
+name: Unit testing
+
+on:
+  pull_request:
+    branches:
+      - develop
+      - main
+    paths:
+      - 'lib/**'
+      - 'uac'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Unit testing
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout uac repository
+        uses: actions/checkout@v4
+        with:
+          repository: tclahr/uac
+          path: uac
+
+      - name: Checkout ushunit repository
+        uses: actions/checkout@v4
+        with:
+          repository: tclahr/ushunit
+          ref: main
+          path: ushunit
+
+      - name: Checkout uac-tests repository
+        uses: actions/checkout@v4
+        with:
+          repository: tclahr/uac-tests
+          ref: ${{ github.event.pull_request.base.ref }}
+          path: uac-tests
+
+      - name: Run tests
+        working-directory: ushunit
+        run: |
+          UAC_DIR="../uac" ./ushunit -i ../uac-tests/tests/lib/*.sh ../uac-tests/tests/*.sh

.github/workflows/validate-artifacts.yaml

@@ -0,0 +1,31 @@
+name: Validate Artifacts
+
+on:
+  pull_request:
+    branches:
+      - develop
+      - main
+    paths:
+      - 'artifacts/**'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  validate-artifact-files:
+    name: Validate Artifacts
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Clone uac repo
+        uses: actions/checkout@v4
+        with:
+          path: uac
+
+      - name: Validate Artifacts
+        working-directory: uac
+        run: |
+          find artifacts/* -name "*.yaml" -type f \
+            | while read file || [ -n "${file}" ]; do
+                ./uac --validate-artifact "${file}"
+              done

.github/workflows/validate-profiles.yaml

@@ -0,0 +1,32 @@
+name: Validate Profiles
+
+on:
+  pull_request:
+    branches:
+      - develop
+      - main
+    paths:
+      - 'profiles/**'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  validate-profile-files:
+    name: Validate Profiles
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Clone uac repo
+        uses: actions/checkout@v4
+        with:
+          path: uac
+
+      - name: Validate Profiles
+        working-directory: uac
+        run: |
+          find profiles/* -name "*.yaml" -type f \
+            | while read file || [ -n "${file}" ]; do
+                ./uac --validate-profile "${file}"
+              done
+      

CHANGELOG.md

@@ -1,13 +1,116 @@
 # Changelog
 
-## 2.9.1 (2024-06-12)
+## 3.0.0 (2024-10-22)
 
-### Fixes
+### Features
 
-- live_response/containers/docker.yaml: Fixed docker stats command that was running in a loop and therefore the program was not terminating [linux] (by [0xtter](https://github.com/0xtter)).
-- live_response/containers/podman.yaml: Fixed docker stats command that was running in a loop and therefore the program was not terminating [linux].
+- New '--enable-modifiers' command line option. Enabling this option will case UAC to run artifacts that change the current system state ([#272](https://github.com/tclahr/uac/issues/272)).
+- UAC now completely skips an artifact file (YAML) that has no artifacts to be collected for the target operating system. You can use '--artifacts list [OPERATING_SYSTEM]' to display artifacts for a specific operating system only.
+- New output file formats:
+  - none: Collected data will not be archived or compressed. Instead, it will be copied directly to an output directory ([#188](https://github.com/tclahr/uac/issues/188)).
+  - zip: Collected data will be archived and compressed into a zip file. Additionally, you can create a password-protected zip file using the '--output-password' option ([#149](https://github.com/tclahr/uac/issues/149)).
+- You can now set a custom output file name using the '-o/--output-base-name' command line option. Variables are available to format the filename ([#179](https://github.com/tclahr/uac/issues/179)).
+- Now you have the option to supply a file path to a custom profile located outside the profiles directory.
+- Now you have the option to supply a file path to a custom artifact located outside the artifacts directory ([#154](https://github.com/tclahr/uac/issues/154)).
+- Now you can have the option to supply a file path to a custom config file located outside the config directory using the '-c/--config' command line option.
+- New remote transfer options for Amazon, Google and IBM cloud storage locations.
+- UAC will now use 'wget' to transfer files to remote cloud storage locations when 'curl' is not available.
+- You can now increase the verbosity level using the '-v/--verbose' command line option. Enabling a higher verbosity level will result in the display of all executed commands.
+- UAC will now use the built-in function 'astrings' to extract strings from binary files when 'strings' is not available on the system.
+- The message 'The strings command requires the command line developer tools.' will no longer appear on macOS systems without developer tools installed ([#171](https://github.com/tclahr/uac/issues/171)).
+- Error messages generated by executed commands (stderr) are now recorded in the uac.log file ([#150](https://github.com/tclahr/uac/issues/150)).
+- New '-H/--hash-collected' command line option. Enabling this option will cause UAC to hash all collected files and save the results in a hash file. To accomplish this, all collected data must first be copied to the destination directory. Therefore, ensure you have twice the free space available on the system: once for the collected data and once for the output file. Additionally, note that this process will increase the running time ([#189](https://github.com/tclahr/uac/issues/189)).
+- You can now validate profiles using the '--validate-profile' command line option.
 
 ### Artifacts
 
-- files/shell/history.yaml: Added collection support for *.historynew files [all].
-- files/shell/sessions.yaml: Added collection support for *.session files [all] [randomaccess3](https://github.com/randomaccess3))
+- bodyfile/bodyfile.yaml: Updated to remove max_depth limit.
+- files/applications/git.yaml: Added collection of files that can be used to run persistence [linux, macos] ([mnrkbys](https://github.com/mnrkbys)).
+- files/applications/lesshst.yaml: Added less history file (.lesshst) collection [aix, freebsd, linux, macos, netbsd, netscaler, openbsd, solaris] ([mnrkbys](https://github.com/mnrkbys)).
+- files/applications/whatsapp.yaml: Added collection of WhatsApp Desktop files [macos].
+- files/logs/additional_logs.yaml: Artifact was renamed to advanced_log_search.yaml.
+- files/logs/relink.yaml: Added collection of the kernel relink log file [openbsd] ([Herbert-Karl](https://github.com/Herbert-Karl)).
+- files/logs/run_log.yaml: Added collection of /run/log directory.
+- files/packages/apt.yaml: Add artifacts to collect package manager plugins/scripts [linux] ([mnrkbys](https://github.com/mnrkbys)).
+- files/packages/dnf.yaml: Add artifacts to collect package manager plugins/scripts [linux] ([mnrkbys](https://github.com/mnrkbys)).
+- files/packages/pkg_contents.yaml: Updated to collect FreeBSD installed packages database [freebsd] ([Herbert-Karl](https://github.com/Herbert-Karl)).
+- files/packages/yum.yaml: Add artifacts to collect package manager plugins/scripts [linux] ([mnrkbys](https://github.com/mnrkbys)).
+- files/system/acct.yaml: Added collection of system accounting files [freebsd, netbsd, openbsd] ([Herbert-Karl](https://github.com/Herbert-Karl)).
+- files/system/acct.yaml: Updated to collect system accounting files [solaris] ([sec-hbaer](https://github.com/sec-hbaer)).
+- files/system/dev_db.yaml: Added collection of the database file used for device lookups [netbsd, openbsd] ([Herbert-Karl](https://github.com/Herbert-Karl)).
+- files/system/dev_shm.yaml: Updated to increase max_file_size to 10MB.
+- files/system/locate_db.yaml: Added collection of the database file used by locate command, representing a snapshot of the virtual file system accessible with minimal permissions [freebsd, netbsd, openbsd] ([Herbert-Karl](https://github.com/Herbert-Karl)).
+- files/system/netscaler.yaml: Updated to increase max_file_size to 10MB.
+- files/system/run_shm.yaml: Updated to increase max_file_size to 10MB.
+- files/system/security_backups.yaml: Added collection of file backups and hashes created by the integrated security script [freebsd, netbsd, openbsd] ([Herbert-Karl](https://github.com/Herbert-Karl)).
+- files/system/systemd.yaml: Updated to add new locations for configuration files.
+- files/system/tmp.yaml: Updated to increase max_file_size to 10MB.
+- files/system/udev.yaml: Added collection of udev rule files ([mnrkbys](https://github.com/mnrkbys)).
+- files/system/var_tmp.yaml: Updated to increase max_file_size to 10MB.
+- hash_executables/hash_executables.yaml: Updated to remove max_depth and max_file_size properties.
+- live_response/containers/jls.yaml: Added collection of jails used on FreeBSD systems [freebsd] ([Herbert-Karl](https://github.com/Herbert-Karl)).
+- live_response/hardware/dmesg.yaml: Updated collection of console message bufffer [esxi, freebsd, netscaler, openbsd, solaris] ([Herbert-Karl](https://github.com/Herbert-Karl)).
+- live_response/modifiers/revel_hidden_processes.yaml: Added command to umount filesystems mounted onto a directory that tipically corresponds to a process ID (PID) [linux] ([halpomeranz](https://github.com/halpomeranz)).
+- live_response/network/procfs_information.yaml: Added collection of TCP and UDP network details from /proc/net [linux]. 
+- live_response/process/deleted.yaml: Collection of deleted processes will no longer use dd conv=swab. The binary file will be collected in its raw format now [linux].
+- live_response/process/deleted.yaml: Updated to fix the collection of open files of (malicious) processes [linux] ([mnrkbys](https://github.com/mnrkbys)).
+- live_response/process/hash_running_processes.yaml: Updated to add support to hash running processes on FreeBSD systems that are using procfs (/proc) [freebsd].
+- live_response/process/procfs_information.yaml: Added artifact collection using cat when strings is not available.
+- live_response/process/procfs_information.yaml: Updated to collect /proc/*/mount [linux] ([halpomeranz](https://github.com/halpomeranz)).
+- live_response/process/procfs_information.yaml: Updated to collect /proc/*/stat [linux] ([mnrkbys](https://github.com/mnrkbys)).
+- live_response/process/strings_running_processes.yaml: Added collection of strings from running processes for ESXi systems [esxi].
+- live_response/process/strings_running_processes.yaml: Added condition to check whether developer tools are installed before running strings on macOS [macos].
+- live_response/process/strings_running_processes.yaml: Added support for collecting strings even when the strings command is unavailable. In such cases, the built-in astrings command will be used instead [all].
+- live_response/storage/btrfs.yaml: Added collection of btrfs mountpoints, subvolumes and snapshots information [linux] ([mnrkbys](https://github.com/mnrkbys)).
+- live_response/system/acctadm.yaml: Added collection of configuration for extended accounting [solaris] ([sec-hbaer](https://github.com/sec-hbaer)).
+- live_response/system/acctcom.yaml: Added collection of the last commands executed in a reverse order based on the default and historic accounting files [solaris] ([sec-hbaer](https://github.com/sec-hbaer)).
+- live_response/system/bpftool.yaml: Added eBPF programs information collection using bpftool [linux] ([mnrkbys](https://github.com/mnrkbys)).
+- live_response/system/hidden_directories.yaml: Updated to remove max_depth limit.
+- live_response/system/hidden_files.yaml: Updated to remove max_depth limit.
+- live_response/system/kernel_tainted_state.yaml: Added collection of dmesg messages showing modules tainting the kernel [linux].
+- live_response/system/lastcomm.yaml: Added collection of the last commands executed in a reverse order based on the default and historic accounting file [freebsd, netbsd, openbsd] ([Herbert-Karl](https://github.com/Herbert-Karl)).
+- live_response/system/lastcomm.yaml: Updated to collect the last commands executed in a reverse order based on the extended accounting file [solaris] ([sec-hbaer](https://github.com/sec-hbaer)).
+- live_response/system/sgid.yaml: Updated to remove max_depth limit.
+- live_response/system/socket_files.yaml: Updated to remove max_depth limit.
+- live_response/system/suid.yaml: Updated to remove max_depth limit.
+- live_response/system/sys_modules.yaml: Removed as it is was duplicate artifact with kernel_modules.yaml.
+- live_response/system/world_writable_directories.yaml: Updated to remove max_depth limit.
+- live_response/system/world_writable_files.yaml: Updated to remove max_depth limit.
+- live_response/system/zoneadm.yaml: Artifact was moved to live_response/containers directory ([Herbert-Karl](https://github.com/Herbert-Karl)).
+
+### Profiles
+
+- files/applications/git.yaml, files/applications/lesshst.yaml, files/applications/viminfo.yaml, and files/applications/wget.yaml artifacts were added to the 'ir_triage' profile.
+
+### Command Line Option Changes
+
+- '--date-range-start' was renamed to '--start-date' ([#186](https://github.com/tclahr/uac/issues/186)).
+- '--date-range-end' was renamed to '--end-date' ([#186](https://github.com/tclahr/uac/issues/186)).
+- '--validate-artifacts-file' was renamed to '--validate-artifact'.
+- '--s3-presigned-url' was renamed to '--aws-s3-presigned-url'.
+- '--s3-presigned-url-log-file' was renamed to '--aws-s3-presigned-url-log-file'.
+- '--ibm-cos-url', '--ibm-cos-url-log-file' and '--ibm-cloud-api-key' were removed and now transfers to IBM cloud should be done using '--s3-provider', '--s3-region', '--s3-bucket' and '--s3-token' options.
+
+### Artifacts Properties Changes
+
+- Introduced a new global 'modifier' property that ensures the artifact runs only if '--enable-modifiers' command line option is used.
+- Introduced a new 'condition' property that ensures the collection runs only if the specified condition returns true.
+- The 'output_directory' property is now mandatory for the following collectors: command, find, hash and stat.
+- The 'file_type' property is now an array.
+- The 'permissions' property is now an array.
+
+### uac.conf
+
+- Introduced a new global 'max_depth' configuration option to limit the depth of directory tree searches globally.
+
+### Tools
+
+- Statically linked 'zip' is now available for the following systems:
+  - linux/esxi (arm, arm64, i386 and x86_64)
+  - freebsd/netscaler (i386 and x86_64)
+- 'avml' and 'linux_procmemdump.sh' tools were moved to the 'bin' directory.
+- AVML updated to v0.14.0.
+
+### Deprecated
+
+- Android support was removed, but UAC can still be executed on Android systems using '--operating-system linux' option.

CODE_OF_CONDUCT.md

@@ -79,4 +79,4 @@ To report incidents or to appeal reports of incidents, send email to iplsdk@linu
 
 ## Credits
 
-This code is based on the [Hyperledger Project's CoC](https://github.com/hyperledger/hyperledger/wiki/Hyperledger-Project-Code-of-Conduct), [W3C’s Code of Ethics and Professional Conduct](https://www.w3.org/Consortium/cepc) with some additions from the [Cloud Foundry](https://www.cloudfoundry.org/)‘s Code of Conduct.
+This code is based on the [Hyperledger Project's CoC](https://github.com/hyperledger/hyperledger/wiki/Hyperledger-Project-Code-of-Conduct), [W3C's Code of Ethics and Professional Conduct](https://www.w3.org/Consortium/cepc) with some additions from the [Cloud Foundry](https://www.cloudfoundry.org/)‘s Code of Conduct.

CONTRIBUTING.md

@@ -8,13 +8,13 @@ Here are a couple of things we are looking for help with:
 
 ## New artifacts
 
-Have you identified a new artifact that is still not collected by UAC? Please create a new artifact file and submit it via a new Pull Request.
+Have you identified a new artifact that is still not collected by UAC? Please create a new artifact and submit it via a new Pull Request.
 
-Please see [Artifacts file definition](https://tclahr.github.io/uac-docs/latest/artifacts_file/) docs for more information.
+Please see [Artifacts definition](https://tclahr.github.io/uac-docs/artifacts/) docs for more information.
 
 ## New features
 
-You can request a new feature by submitting an issue to our GitHub Repository. If you would like to implement a new feature, please submit an issue with a proposal for your work first, to be sure that we can use it. This will also allow us to better coordinate our efforts, prevent duplication of work, and help you to craft the change so that it is successfully accepted into the project.
+You can request a new feature by submitting an issue to our GitHub Repository. If you would like to implement a new feature, please submit an issue with a proposal for your work first, to be sure that we can use it. This will also allow us to better coordinate our efforts, prevent duplication of work, and help you craft the change so that it is successfully accepted into the project.
 
 ## Found a bug?
 
@@ -84,9 +84,9 @@ git checkout -b my-feature-branch develop
 
 1. Create your code following our [Coding Rules](#coding-rules).
 
-1. Test your code against as many systems as you can using the [uac-unit-test](https://github.com/tclahr/uac-unit-test). For instance, your code can fully work on a Linux but not on a FreeBSD system.
+1. Test your code against as many systems as you can. For instance, your code can fully work on a Linux but not on a FreeBSD system.
 
-1. Commit your changes using a descriptive commit message that follows our [commit message guidelines](#commit-message-guidelines). *Don’t commit code as an unrecognized author. Having commits with unrecognized authors makes it more difficult to track who wrote which part of the code. Ensure your Git client is configured with the correct email address and linked to your GitHub user.*
+1. Commit your changes using a descriptive commit message that follows our [commit message guidelines](#commit-message-guidelines). *Don't commit code as an unrecognized author. Having commits with unrecognized authors makes it more difficult to track who wrote which part of the code. Ensure your Git client is configured with the correct email address and linked to your GitHub user.*
 
   ```shell
   git commit -s

LICENSE

@@ -1,4 +1,3 @@
-
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/