Worked on scripts and tests

.github/workflows/test_docker.yml

@@ -0,0 +1,84 @@
+# Run tests on Fedora and Ubuntu Docker images using GIFT CORP and GIFT PPA on commit
+name: test_docker
+on: [push]
+permissions: read-all
+jobs:
+  test_fedora:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: ['39']
+    container:
+      image: registry.fedoraproject.org/fedora:${{ matrix.version }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up container
+      run: |
+        dnf install -y dnf-plugins-core langpacks-en
+    - name: Install dependencies
+      run: |
+        dnf copr -y enable @gift/dev
+        dnf install -y @development-tools python3 python3-devel libbde-python3 libcaes-python3 libcreg-python3 libewf-python3 libexe-python3 libfcrypto-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libfwps-python3 libfwsi-python3 libhmac-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 libwrc-python3 python3-acstore python3-artifacts python3-cffi python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-dtfabric python3-idna python3-pytsk3 python3-pyyaml python3-setuptools python3-xattr
+    - name: Run tests
+      env:
+        LANG: C.utf8
+      run: |
+        python3 ./run_tests.py
+    - name: Run end-to-end tests
+      run: |
+        if test -f tests/end-to-end.py; then PYTHONPATH=. python3 ./tests/end-to-end.py --debug -c config/end-to-end.ini; fi
+    - name: Build source distribution
+      run: |
+        python3 ./setup.py sdist
+    - name: Build binary distribution
+      run: |
+        python3 ./setup.py bdist
+    - name: Run build and install test
+      run: |
+        python3 ./setup.py build
+        python3 ./setup.py install
+  test_ubuntu:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: ['22.04']
+    container:
+      image: ubuntu:${{ matrix.version }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up container
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        apt-get update -q
+        apt-get install -y libterm-readline-gnu-perl locales software-properties-common
+        locale-gen en_US.UTF-8
+        ln -f -s /usr/share/zoneinfo/UTC /etc/localtime
+    - name: Install dependencies
+      run: |
+        add-apt-repository -y ppa:gift/dev
+        apt-get update -q
+        apt-get install -y build-essential python3 python3-dev libbde-python3 libcaes-python3 libcreg-python3 libewf-python3 libexe-python3 libfcrypto-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libfwps-python3 libfwsi-python3 libhmac-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 libwrc-python3 python3-acstore python3-artifacts python3-cffi-backend python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-setuptools python3-wheel python3-xattr python3-yaml
+    - name: Run tests
+      env:
+        LANG: en_US.UTF-8
+      run: |
+        python3 ./run_tests.py
+    - name: Run end-to-end tests
+      env:
+        LANG: en_US.UTF-8
+      run: |
+        if test -f tests/end-to-end.py; then PYTHONPATH=. python3 ./tests/end-to-end.py --debug -c config/end-to-end.ini; fi
+    - name: Update setuptools
+      run: |
+        python3 -m pip install -U setuptools
+    - name: Build source distribution
+      run: |
+        python3 ./setup.py sdist
+    - name: Build binary distribution
+      run: |
+        python3 ./setup.py bdist
+    - name: Run build and install test
+      run: |
+        python3 ./setup.py build
+        python3 ./setup.py install

.github/workflows/test_docs.yml

@@ -0,0 +1,47 @@
+# Run docs tox tests on Ubuntu Docker images using GIFT PPA
+name: test_docs
+on:
+  pull_request:
+    branches:
+    - main
+  push:
+    branches:
+    - main
+permissions: read-all
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+        - python-version: '3.12'
+          toxenv: 'docs'
+    container:
+      image: ubuntu:22.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up container
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        apt-get update -q
+        apt-get install -y libterm-readline-gnu-perl locales software-properties-common
+        locale-gen en_US.UTF-8
+        ln -f -s /usr/share/zoneinfo/UTC /etc/localtime
+    - name: Install dependencies
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        add-apt-repository -y universe
+        add-apt-repository -y ppa:deadsnakes/ppa
+        add-apt-repository -y ppa:gift/dev
+        apt-get update -q
+        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcaes-python3 libcreg-python3 libewf-python3 libexe-python3 libfcrypto-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libfwps-python3 libfwsi-python3 libhmac-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 libwrc-python3 python3-acstore python3-artifacts python3-cffi-backend python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-setuptools python3-xattr python3-yaml
+    - name: Install tox
+      run: |
+        python3 -m pip install tox
+    - name: Run tests
+      env:
+        LANG: en_US.UTF-8
+      run: |
+        tox -e${{ matrix.toxenv }}

.github/workflows/test_tox.yml

@@ -0,0 +1,129 @@
+# Run tox tests on Ubuntu Docker images using GIFT PPA
+name: test_tox
+on:
+  pull_request:
+    branches:
+    - main
+  push:
+    branches:
+    - main
+permissions: read-all
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+        - python-version: '3.8'
+          toxenv: 'py38,wheel'
+        - python-version: '3.9'
+          toxenv: 'py39,wheel'
+        - python-version: '3.10'
+          toxenv: 'py310,wheel'
+        - python-version: '3.11'
+          toxenv: 'py311,wheel'
+        - python-version: '3.12'
+          toxenv: 'py312,wheel'
+    container:
+      image: ubuntu:22.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up container
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        apt-get update -q
+        apt-get install -y libterm-readline-gnu-perl locales software-properties-common
+        locale-gen en_US.UTF-8
+        ln -f -s /usr/share/zoneinfo/UTC /etc/localtime
+    - name: Install dependencies
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        add-apt-repository -y universe
+        add-apt-repository -y ppa:deadsnakes/ppa
+        add-apt-repository -y ppa:gift/dev
+        apt-get update -q
+        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcaes-python3 libcreg-python3 libewf-python3 libexe-python3 libfcrypto-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libfwps-python3 libfwsi-python3 libhmac-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 libwrc-python3 python3-acstore python3-artifacts python3-cffi-backend python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-setuptools python3-xattr python3-yaml
+    - name: Install tox
+      run: |
+        python3 -m pip install tox
+    - name: Run tests
+      env:
+        LANG: en_US.UTF-8
+      run: |
+        tox -e${{ matrix.toxenv }}
+  coverage:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+        - python-version: '3.10'
+          toxenv: 'coverage'
+    container:
+      image: ubuntu:22.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up container
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        apt-get update -q
+        apt-get install -y libterm-readline-gnu-perl locales software-properties-common
+        locale-gen en_US.UTF-8
+        ln -f -s /usr/share/zoneinfo/UTC /etc/localtime
+    - name: Install dependencies
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        add-apt-repository -y universe
+        add-apt-repository -y ppa:deadsnakes/ppa
+        add-apt-repository -y ppa:gift/dev
+        apt-get update -q
+        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcaes-python3 libcreg-python3 libewf-python3 libexe-python3 libfcrypto-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libfwps-python3 libfwsi-python3 libhmac-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 libwrc-python3 python3-acstore python3-artifacts python3-cffi-backend python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-setuptools python3-xattr python3-yaml
+    - name: Install tox
+      run: |
+        python3 -m pip install tox
+    - name: Run tests with coverage
+      env:
+        LANG: en_US.UTF-8
+      run: |
+        tox -e${{ matrix.toxenv }}
+    - name: Upload coverage report to Codecov
+      uses: codecov/codecov-action@v3
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+        - python-version: '3.12'
+          toxenv: 'lint'
+    container:
+      image: ubuntu:22.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up container
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        apt-get update -q
+        apt-get install -y libterm-readline-gnu-perl locales software-properties-common
+        locale-gen en_US.UTF-8
+        ln -f -s /usr/share/zoneinfo/UTC /etc/localtime
+    - name: Install dependencies
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        add-apt-repository -y universe
+        add-apt-repository -y ppa:deadsnakes/ppa
+        add-apt-repository -y ppa:gift/dev
+        apt-get update -q
+        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcaes-python3 libcreg-python3 libewf-python3 libexe-python3 libfcrypto-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libfwps-python3 libfwsi-python3 libhmac-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 libwrc-python3 python3-acstore python3-artifacts python3-cffi-backend python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-setuptools python3-xattr python3-yaml
+    - name: Install tox
+      run: |
+        python3 -m pip install tox
+    - name: Run linter
+      env:
+        LANG: en_US.UTF-8
+      run: |
+        tox -e${{ matrix.toxenv }}

appveyor.yml

@@ -0,0 +1,54 @@
+environment:
+  matrix:
+  - DESCRIPTION: "Run tests on Windows with 32-bit Python 3.12"
+    MACHINE_TYPE: "x86"
+    APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+    PYTHON: "C:\\Python312"
+    PYTHON_VERSION: "3.12"
+    L2TBINARIES_TRACK: "dev"
+    TARGET: tests
+  - DESCRIPTION: "Run tests on Windows with 64-bit Python 3.12"
+    MACHINE_TYPE: "amd64"
+    APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+    PYTHON: "C:\\Python312-x64"
+    PYTHON_VERSION: "3.12"
+    L2TBINARIES_TRACK: "dev"
+    TARGET: tests
+  - DESCRIPTION: "Build wheel on Windows with 32-bit Python 3.12"
+    MACHINE_TYPE: "amd64"
+    APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+    PYTHON: "C:\\Python312-x64"
+    PYTHON_VERSION: "3.12"
+    L2TBINARIES_TRACK: "dev"
+    TARGET: wheel
+  - DESCRIPTION: "Build wheel on Windows with 64-bit Python 3.12"
+    MACHINE_TYPE: "amd64"
+    APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+    PYTHON: "C:\\Python312-x64"
+    PYTHON_VERSION: "3.12"
+    L2TBINARIES_TRACK: "dev"
+    TARGET: wheel
+  - DESCRIPTION: "Run tests on Mac OS with Python 3.12"
+    APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
+    HOMEBREW_NO_INSTALL_CLEANUP: 1
+    TARGET: tests
+
+install:
+- cmd: "%PYTHON%\\python.exe -m pip install -U build pip setuptools twine wheel"
+- ps: If ($isWindows) { .\config\appveyor\install.ps1 }
+- sh: config/appveyor/install.sh
+
+build_script:
+- cmd: IF [%TARGET%]==[wheel] (
+    "%PYTHON%\\python.exe" -m build --wheel )
+
+test_script:
+- cmd: IF [%TARGET%]==[tests] (
+    "%PYTHON%\\python.exe" run_tests.py &&
+    IF EXIST "tests\\end-to-end.py" (
+        set PYTHONPATH=. &&
+        "%PYTHON%\\python.exe" "tests\\end-to-end.py" --debug -c "config\\end-to-end.ini" ) )
+- sh: config/appveyor/runtests.sh
+
+artifacts:
+- path: dist\*.whl

config/appveyor/install.ps1

@@ -0,0 +1,27 @@
+# Script to set up tests on AppVeyor Windows.
+
+$Dependencies = "PyYAML acstore artifacts cffi dfdatetime dfimagetools dfvfs dfwinreg dtfabric idna libbde libcaes libcreg libewf libexe libfcrypto libfsapfs libfsext libfsfat libfshfs libfsntfs libfsxfs libfvde libfwnt libfwps libfwsi libhmac libluksde libmodi libphdi libqcow libregf libsigscan libsmdev libsmraw libvhdi libvmdk libvsapm libvsgpt libvshadow libvslvm libwrc pytsk3 xattr"
+
+If ($Dependencies.Length -gt 0)
+{
+	$Dependencies = ${Dependencies} -split " "
+
+	$Output = Invoke-Expression -Command "git clone https://github.com/log2timeline/l2tdevtools.git ..\l2tdevtools 2>&1" | %{ "$_" }
+	Write-Host (${Output} | Out-String)
+
+	If ($env:APPVEYOR_REPO_BRANCH -eq "main")
+	{
+		$Track = "stable"
+	}
+	Else
+	{
+		$Track = $env:APPVEYOR_REPO_BRANCH
+	}
+	New-Item -ItemType "directory" -Name "dependencies"
+
+	$env:PYTHONPATH = "..\l2tdevtools"
+
+	$Output = Invoke-Expression -Command "& '${env:PYTHON}\python.exe' ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type ${env:MACHINE_TYPE} --track ${env:L2TBINARIES_TRACK} ${Dependencies} 2>&1" | %{ "$_" }
+	Write-Host (${Output} | Out-String)
+}
+

config/appveyor/install.sh

@@ -0,0 +1,7 @@
+# Script to set up tests on AppVeyor MacOS.
+
+set -e
+
+brew update -q
+brew install -q gettext gnu-sed python@3.12 tox || true
+

config/appveyor/runtests.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+# Script to run tests
+
+# Set the following environment variables to build libyal with gettext.
+export CPPFLAGS="-I/usr/local/include -I/usr/local/opt/gettext/include ${CPPFLAGS}";
+export LDFLAGS="-L/usr/local/lib -L/usr/local/opt/gettext/lib ${LDFLAGS}";
+
+# Set the following environment variables to build pycrypto and yara-python.
+export CPPFLAGS="-I/usr/local/opt/openssl@1.1/include ${CPPFLAGS}";
+export LDFLAGS="-L/usr/local/opt/openssl@1.1/lib ${LDFLAGS}";
+
+# Set the following environment variables to ensure tox can find Python 3.12.
+export PATH="/usr/local/opt/python@3.12/bin:${PATH}";
+
+tox -e py312