diff --git a/.github/workflows/ce-dev-PR-test.yml b/.github/workflows/ce-dev-PR-test.yml index cb3be05..a291e70 100644 --- a/.github/workflows/ce-dev-PR-test.yml +++ b/.github/workflows/ce-dev-PR-test.yml @@ -30,5 +30,6 @@ jobs: rm -Rf mkcert - name: Build and test run: | - /bin/sh docker-images/export.sh latest + /bin/sh docker-images/export.sh --version latest --image-name ce-dev + /bin/sh docker-images/export.sh --version latest --image-name ce-dev-controller /bin/sh templates/prebuild.sh diff --git a/.github/workflows/ce-dev-build-dev.yml b/.github/workflows/ce-dev-build-dev.yml index ac63636..7f592bf 100644 --- a/.github/workflows/ce-dev-build-dev.yml +++ b/.github/workflows/ce-dev-build-dev.yml @@ -39,4 +39,5 @@ jobs: - name: Build and push Docker images run: | echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin - /bin/sh docker-images/export.sh devel --push + /bin/sh docker-images/export.sh --version devel --image-name ce-dev --base-image debian:bookworm-slim --dockerfile-path base-devel --push + /bin/sh docker-images/export.sh --version devel --image-name ce-dev-controller --base-image debian:bookworm-slim --dockerfile-path controller-devel --push diff --git a/.github/workflows/ce-dev-test.yml b/.github/workflows/ce-dev-test.yml index 9eb8fdc..0652fcc 100644 --- a/.github/workflows/ce-dev-test.yml +++ b/.github/workflows/ce-dev-test.yml @@ -34,5 +34,6 @@ jobs: rm -Rf mkcert - name: Build and test run: | - /bin/sh docker-images/export.sh latest + /bin/sh docker-images/export.sh --version latest --image-name ce-dev + /bin/sh docker-images/export.sh --version latest --image-name ce-dev-controller /bin/sh templates/prebuild.sh diff --git a/.gitignore b/.gitignore index e30f02b..409c021 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ lib tmp node_modules oclif.manifest.json +.vscode/* diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 0bfb6d2..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "editor.codeActionsOnSave": { - "source.fixAll.tslint": true - } -} \ No newline at end of file diff --git a/docker-images/base-devel/Dockerfile b/docker-images/base-devel/Dockerfile new file mode 100644 index 0000000..218300b --- /dev/null +++ b/docker-images/base-devel/Dockerfile @@ -0,0 +1,103 @@ +FROM debian:bookworm-slim as unison +RUN \ + set -x && \ + export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y -o Dpkg::Options::="--force-confnew" \ + p7zip-full \ + build-essential \ + wget \ + ocaml-native-compilers && \ + cd /tmp/ && \ + wget https://github.com/bcpierce00/unison/archive/v2.53.3.tar.gz && \ + tar -xzvf v2.53.3.tar.gz && \ + cd /tmp/unison-2.53.3 && \ + make + +FROM unison + +RUN \ + set -x && \ + export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get dist-upgrade -y -o Dpkg::Options::="--force-confnew" && \ + apt-get install -y -o Dpkg::Options::="--force-confnew" \ + anacron \ + apt-transport-https \ + apt-utils \ + aptitude \ + bash \ + binutils \ + cron \ + curl \ + dirmngr \ + gnupg \ + rsync \ + openssh-server \ + postfix \ + procmail \ + python3-apt \ + python3-dev \ + python3-pycurl \ + python3-pip \ + python3-venv \ + rsyslog \ + sudo \ + systemd \ + systemd-sysv \ + unzip \ + vim \ + wget && \ + apt-get clean && \ + update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \ + rm -rf \ + /var/lib/apt/lists/* \ + /var/log/* \ + /tmp/* + +RUN \ + echo 'UseDNS no' >> /etc/ssh/sshd_config && \ + mkdir -p /var/run/sshd && \ + rm /usr/sbin/policy-rc.d + +RUN \ + set -x && \ + export DEBIAN_FRONTEND=noninteractive && \ + useradd -s /bin/bash ce-dev && \ + echo ce-dev:ce-dev | chpasswd -m && \ + install -m 755 -o ce-dev -g ce-dev -d /home/ce-dev && \ + install -m 700 -o ce-dev -g ce-dev -d /home/ce-dev/.ssh && \ + echo root:ce-dev | chpasswd -m && \ + echo 'ce-dev ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/ce-dev && \ + chmod 0440 /etc/sudoers.d/ce-dev && \ + rm -rf /tmp/* + +RUN \ + rm -f \ + /etc/machine-id \ + /var/lib/dbus/machine-id + +COPY --from=unison /tmp/unison-2.53.3/src/unison /usr/local/bin/ +COPY --from=unison /tmp/unison-2.53.3/src/unison-fsmonitor /usr/local/bin/ +COPY ./ce-dev-ownership.sh /opt/ +COPY ./ce-dev-ssh.sh /opt/ +COPY ./unison.sh /opt/ +COPY ./unison-startup.sh /opt/ +COPY ./procmailrc /etc/procmailrc + +RUN \ + wget https://github.com/FiloSottile/mkcert/releases/download/v1.4.4/mkcert-v1.4.4-linux-amd64 -O /usr/local/bin/mkcert && \ + mkdir -p /home/ce-dev/deploy/live.local /home/ce-dev/.composer/cache /home/ce-dev/.nvm/versions/node /home/ce-dev/.local/share/mkcert && \ + chown -R ce-dev:ce-dev /home/ce-dev && \ + chmod +x /usr/local/bin/* + +RUN \ + systemctl mask -- \ + dev-hugepages.mount \ + sys-fs-fuse-connections.mount + +ENV container docker +STOPSIGNAL SIGRTMIN+3 +VOLUME [ "/sys/fs/cgroup", "/run", "/run/lock", "/tmp" ] + +ENTRYPOINT ["/sbin/init"] \ No newline at end of file diff --git a/docker-images/base-devel/ce-dev-ownership.sh b/docker-images/base-devel/ce-dev-ownership.sh new file mode 100644 index 0000000..3df76ce --- /dev/null +++ b/docker-images/base-devel/ce-dev-ownership.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +# Ensure user numeric uid/gid matches. +# @param $1 +# User id. +# @param $2 +# Group id. +ensure_user_ids(){ + OLD_UID="$(id -u ce-dev)" + OLD_GID="$(id -g ce-dev)" + if [ "$OLD_UID" = "$1" ] && [ "$OLD_GID" = "$2" ]; then + return + fi + if [ "$OLD_UID" != "$1" ]; then + usermod -u "$1" ce-dev + chown -R --from="$OLD_UID" "$1" /var + echo "User ID changed to $1." + fi + if [ "$OLD_GID" != "$2" ]; then + groupmod -g "$2" ce-dev + chown -R --from=":$OLD_GID" ":$2" /var + echo "Group ID changed to $2." + fi + if [ -d /.x-ce-dev ]; then + chown -R ce-dev:ce-dev /.x-ce-dev + fi + chown -R ce-dev:ce-dev /home/ce-dev +} + +# Match ids with host user. +if [ -n "$1" ] && [ -n "$2" ]; then + ensure_user_ids "$1" "$2" +fi \ No newline at end of file diff --git a/docker-images/base-devel/ce-dev-ssh.sh b/docker-images/base-devel/ce-dev-ssh.sh new file mode 100644 index 0000000..67b642c --- /dev/null +++ b/docker-images/base-devel/ce-dev-ssh.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# Generate ssh key pair. +ensure_ssh_key(){ + rm -rf /home/ce-dev/.ssh/* + ssh-keygen -t rsa -b 4096 -N "" -f /home/ce-dev/.ssh/id_rsa + cp /home/ce-dev/.ssh/id_rsa.pub /home/ce-dev/.ssh/authorized_keys + touch /home/ce-dev/.ssh/config + chmod 600 /home/ce-dev/.ssh/id_rsa + chmod 600 /home/ce-dev/.ssh/id_rsa.pub + chmod 600 /home/ce-dev/.ssh/authorized_keys + chown -R ce-dev:ce-dev /home/ce-dev/.ssh +} + +ensure_ssh_key \ No newline at end of file diff --git a/docker-images/base-devel/procmailrc b/docker-images/base-devel/procmailrc new file mode 100644 index 0000000..bad6ce9 --- /dev/null +++ b/docker-images/base-devel/procmailrc @@ -0,0 +1,3 @@ +ORGMAIL=/dev/null +DEFAULT=${ORGMAIL} +MAILDIR=${ORGMAIL} \ No newline at end of file diff --git a/docker-images/base-devel/unison-startup.sh b/docker-images/base-devel/unison-startup.sh new file mode 100644 index 0000000..28537a0 --- /dev/null +++ b/docker-images/base-devel/unison-startup.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +nohup /bin/sh /opt/unison.sh "$@" /dev/null 2>&1 & \ No newline at end of file diff --git a/docker-images/base-devel/unison.sh b/docker-images/base-devel/unison.sh new file mode 100644 index 0000000..4856ebb --- /dev/null +++ b/docker-images/base-devel/unison.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +mkdir -p "/tmp/$1" +while true; do + flock -w 30 /tmp"$1"/unison.lock /usr/local/bin/unison -copythreshold 100000 -owner -group -batch -repeat watch -fastercheckUNSAFE -prefer "$1" "$@" || exit 1 +done \ No newline at end of file diff --git a/docker-images/controller-devel/Dockerfile b/docker-images/controller-devel/Dockerfile new file mode 100644 index 0000000..469e5c2 --- /dev/null +++ b/docker-images/controller-devel/Dockerfile @@ -0,0 +1,34 @@ +FROM codeenigma/ce-dev-1.x:devel + +RUN \ + set -x && \ + export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get dist-upgrade -y -o Dpkg::Options::="--force-confnew" && \ + apt-get install -y -o Dpkg::Options::="--force-confnew" \ + git \ + python3-venv \ + python3-boto3 && \ + apt-get clean && \ + update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \ + rm -rf \ + /var/lib/apt/lists/* \ + /var/log/* \ + /tmp/* + + +RUN su - ce-dev -c "git clone --branch 2.x https://github.com/codeenigma/ce-provision.git /home/ce-dev/ce-provision" + +COPY ./provision.yml /home/ce-dev/ce-provision/provision.yml + +RUN \ + set -x && \ + export DEBIAN_FRONTEND=noninteractive && \ + su - ce-dev -c "/usr/bin/python3 -m venv /home/ce-dev/ansible" && \ + su - ce-dev -c "/home/ce-dev/ansible/bin/python3 -m pip install ansible" + +RUN \ + set -x && \ + export DEBIAN_FRONTEND=noninteractive && \ + su - ce-dev -c "/home/ce-dev/ansible/bin/ansible-playbook /home/ce-dev/ce-provision/provision.yml" && \ + rm /home/ce-dev/ce-provision/provision.yml diff --git a/docker-images/controller-devel/provision.yml b/docker-images/controller-devel/provision.yml new file mode 100644 index 0000000..e109caa --- /dev/null +++ b/docker-images/controller-devel/provision.yml @@ -0,0 +1,64 @@ +--- +- hosts: localhost + become: true + vars: + - _domain_name: example.com + - _ce_provision_build_tmp_dir: /tmp + - _ce_provision_data_dir: /tmp + - _ce_provision: + username: ce-dev + - is_local: true + - _env_type: utility + - ce_deploy: + venv_path: "/home/{{ _ce_provision.username }}/ansible" + venv_command: /usr/bin/python3 -m venv + install_username: "{{ _ce_provision.username }}" + upgrade_timer_name: upgrade_ce_deploy_ansible + own_repository: https://github.com/codeenigma/ce-deploy.git + config_repository: https://github.com/codeenigma/ce-dev-ce-deploy-config.git + own_repository_branch: 1.x + config_repository_branch: 1.x + username: ce-dev + new_user: true + key_name: id_rsa.pub + local_dir: /home/ce-dev/ce-deploy + ce_provision_dir: "/home/ce-dev/ce-provision" + groups: [] + galaxy_custom_requirements_file: "" + upgrade_galaxy: + enabled: false + - ce_provision: + venv_path: "/home/{{ _ce_provision.username }}/ansible" + venv_command: /usr/bin/python3 -m venv + install_username: "{{ _ce_provision.username }}" + upgrade_timer_name: upgrade_ce_provision_ansible + own_repository: https://github.com/codeenigma/ce-provision.git + own_repository_branch: 2.x + own_repository_skip_checkout: false + config_repository: https://github.com/codeenigma/ce-dev-ce-provision-config.git + config_repository_branch: 1.x + config_repository_skip_checkout: false + username: ce-dev + new_user: true + key_name: id_rsa.pub + local_dir: /home/ce-dev/ce-provision + groups: [] + contrib_roles: + - directory: wazuh + repo: https://github.com/wazuh/wazuh-ansible.git + branch: stable + - directory: systemd_timers + repo: https://github.com/vlcty/ansible-systemd-timers.git + branch: master + galaxy_custom_requirements_file: "" + galaxy_roles_directory: "/home/{{ _ce_provision.username }}/.ansible/roles" + upgrade_galaxy: + enabled: false + - ce_ansible: + upgrade: + enabled: false + linters: + enabled: true + roles: + - debian/ce_provision + - debian/ce_deploy diff --git a/docker-images/export.sh b/docker-images/export.sh index 921f0d8..1b44637 100755 --- a/docker-images/export.sh +++ b/docker-images/export.sh @@ -4,46 +4,106 @@ # usage(){ - cat << EOF -usage: - -Export a base CodeEnigma image, optionally pushing it do Docker Hub. -$0 [--push] + echo 'export.sh [OPTIONS] --version --image-name --push' + echo 'Export a base Code Enigma image, optionally pushing it to your Docker repository.' + echo '' + echo 'Mandatory arguments:' + echo '--version: Version tag to apply to the Docker image, e.g. "latest".' + echo '--image-name: Name of the resulting Docker image, e.g. "ce-dev".' + echo '' + echo 'Available options:' + echo '--push: Push the built image to the Docker repository.' + echo '--base-image: Name of the base image to use, IMPORTANT: must match your Dockerfile - defaults to "debian:bullseye-slim".' + echo '--dockerfile-path: Pass the path within docker-images to your Dockerfile and other build assets - defaults to "base".' + echo '--docker-repo: Pass the Docker repository name - defaults to "codeenigma".' + echo '--ce-dev-version: The version to append to the image name - defaults to "1.x".' +} -EOF +# Parse options arguments. +parse_options(){ + while [ "${1:-}" ]; do + case "$1" in + "--version") + shift + VERSION="$1" + ;; + "--image-name") + shift + IMAGE_NAME="$1" + ;; + "--dockerfile-path") + shift + DOCKERFILE_PATH="$1" + ;; + "--dockerfile-repo") + shift + DOCKER_REPO="$1" + ;; + "--base-image") + shift + BASE_IMAGE="$1" + ;; + "--ce-dev-version") + shift + CE_DEV_VERSION="$1" + ;; + "--push") + PUSH="yes" + ;; + *) + usage + exit 1 + ;; + esac + shift + done } -# Quick check we have args. -if [ -z "$1" ]; then - usage - exit 1; -fi +# Default variables. +DOCKERFILE_PATH="base" +PUSH="no" +BASE_IMAGE="debian:bullseye-slim" +DOCKER_REPO="codeenigma" +CE_DEV_VERSION="1.x" +VERSION="" +IMAGE_NAME="" # Keep current dir in mind to know where to move back when done. -OWN=$(readlink "$0"); +OWN=$(readlink "$0") if [ -z "$OWN" ]; then OWN="$0" fi OWN_DIR=$( cd "$( dirname "$OWN" )" && pwd -P) +# Parse options. +parse_options "$@" + +# Check we have enough arguments. +if [ -z "$VERSION" ] || [ -z "$IMAGE_NAME" ]; then + usage + exit 1 +fi + # Ensure we have a fresh image to start with. -docker image pull debian:bookworm-slim - -# Build base image. -echo "Building base image." -docker image build --compress "--label=ce-dev-1.x:$1" --no-cache=true -t "codeenigma/ce-dev-1.x:$1" "$OWN_DIR/base" || exit 1 -if [ "$2" = "--push" ]; then - echo "Publishing the image with docker image push codeenigma/ce-dev-1.x:$1" - docker image push "codeenigma/ce-dev-1.x:$1" +docker image pull "$BASE_IMAGE" + +# Build image. +echo "Building $DOCKERFILE_PATH image." +docker image build --compress "--label=$IMAGE_NAME-$CE_DEV_VERSION:$VERSION" --no-cache=true -t "$DOCKER_REPO/$IMAGE_NAME-$CE_DEV_VERSION:$VERSION" "$OWN_DIR/$DOCKERFILE_PATH" || exit 1 +if [ $PUSH = "yes" ]; then + echo "Publishing the image with docker image push $DOCKER_REPO/$IMAGE_NAME-$CE_DEV_VERSION:$VERSION" + docker image push "$DOCKER_REPO/$IMAGE_NAME-$CE_DEV_VERSION:$VERSION" fi +# TODO - remove this + # Build controller image. -echo "Building controller image" -docker image build --compress "--label=ce-dev-controller-1.x:$1" --no-cache=true -t "codeenigma/ce-dev-controller-1.x:$1" "$OWN_DIR/controller" || exit 1 -if [ "$2" = "--push" ]; then - echo "Publishing the image with docker image push codeenigma/ce-dev-controller-1.x:$1" - docker image push "codeenigma/ce-dev-controller-1.x:$1" -fi +#echo "Building controller image" +#docker image build --compress "--label=ce-dev-controller-1.x:$1" --no-cache=true -t "codeenigma/ce-dev-controller-1.x:$1" "$OWN_DIR/controller" || exit 1 +#if [ "$2" = "--push" ]; then +# echo "Publishing the image with docker image push codeenigma/ce-dev-controller-1.x:$1" +# docker image push "codeenigma/ce-dev-controller-1.x:$1" +#fi # Build DinD (Docker in Docker) image. # echo "Building DinD image"