From ee40c5e60dce7f3463023f54b8dffaa8d63ad384 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 6 Jun 2024 13:39:19 +0100 Subject: [PATCH] Add devcontainer (#341) * add docker build and devcontainer * add vscode config --------- Co-authored-by: Gary Yendell --- .devcontainer/devcontainer.json | 55 ++++++++++++++++++ .devcontainer/initializeCommand | 6 ++ .devcontainer/postCreateCommand | 28 ++++++++++ .dockerignore | 18 ++++++ .github/workflows/container.yml | 99 +++++++++++++++++++++++++++++++++ .gitignore | 2 +- .vscode/extensions.json | 9 +++ .vscode/launch.json | 17 ++++++ .vscode/settings.json | 20 +++++++ .vscode/tasks.json | 18 ++++++ Dockerfile | 70 +++++++++++++++++++++++ 11 files changed, 341 insertions(+), 1 deletion(-) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/initializeCommand create mode 100644 .devcontainer/postCreateCommand create mode 100644 .dockerignore create mode 100644 .github/workflows/container.yml create mode 100644 .vscode/extensions.json create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json create mode 100644 .vscode/tasks.json create mode 100644 Dockerfile diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..eab87b9f5 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,55 @@ +// For format details, see https://containers.dev/implementors/json_reference/ +{ + "name": "odin-data devcontainer", + "build": { + "dockerfile": "../Dockerfile", + "context": "..", + "target": "developer" + }, + "features": { + // add quality of life features for developers including git config integration + // note this is slow for the odin-data container - leaving out for now + "ghcr.io/devcontainers/features/common-utils:2": { + // don't upgrade to make this similar to the runtime container + "upgradePackages": false + } + }, + // IMPORTANT for this devcontainer to work with docker VSCODE_REMOTE_USER must be + // set to vscode. You will run as vscode with full sudo rights. + // For podman it should be left blank. You will run as root but host mounts + // will be owned by your user. + "remoteUser": "${localEnv:VSCODE_REMOTE_USER}", + "customizations": { + "vscode": { + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.vscode-pylance", + "tamasfe.even-better-toml", + "redhat.vscode-yaml", + "ryanluker.vscode-coverage-gutters", + "epicsdeb.vscode-epics", + "charliermarsh.ruff", + "ms-vscode.cmake-tools", + "ms-vscode.cpptools" + ] + } + }, + // You can place any outside of the container before-launch commands here + "initializeCommand": "bash .devcontainer/initializeCommand ${devcontainerId}", + // One time global setup commands inside the container + "postCreateCommand": "bash .devcontainer/postCreateCommand ${devcontainerId}", + "runArgs": [ + // Allow the container to access the host X11 display and EPICS CA + "--net=host", + // Make sure SELinux does not disable write access to host filesystems like tmp + "--security-opt=label=disable" + ], + // Mount the parent of the project folder so we can access peer projects + "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind", + // mount in other useful files from the host + "mounts": [ + // add extra mounts below + // "source=${localWorkspaceFolder},target=/odin-data,type=bind" + "source=/dev/shm,target=/dev/shm,type=bind" + ] +} \ No newline at end of file diff --git a/.devcontainer/initializeCommand b/.devcontainer/initializeCommand new file mode 100644 index 000000000..732ec1f26 --- /dev/null +++ b/.devcontainer/initializeCommand @@ -0,0 +1,6 @@ +#!/bin/bash + +# custom initialization goes here - runs outside of the dev container +# just before the container is launched but after the container is created + +echo "devcontainerID ${1}" diff --git a/.devcontainer/postCreateCommand b/.devcontainer/postCreateCommand new file mode 100644 index 000000000..156a8f5eb --- /dev/null +++ b/.devcontainer/postCreateCommand @@ -0,0 +1,28 @@ +#!/bin/bash + +# Custom initialization goes here if needed. +# Runs inside the dev container after the container is created + +################################################################################ +# When using docker we will not be root inside the container +# You may wish to change ownership of files you want the user to modify +################################################################################ + +# if [[ $USER != "root" ]] ; then +# # make sure the non-root user can build iocs and (mounted in) support modules +# # sudo chown -R ${USER}:${USER} add_folders_here_if needed +# fi + +################################################################################ +# Shell customizations for Generic IOC devcontainers +################################################################################ + +# pick a zsh theme that does not cause completion corruption in zsh vscode terminals +sed -i $HOME/.zshrc -e 's/ZSH_THEME="devcontainers"/ZSH_THEME="dst"/' + +# allow personalization of all devcontainers in this subdirectory +# by placing a .devcontainer_rc file in the workspace root +if [[ -f /workspaces/.devcontainer_rc ]] ; then + source /workspaces/.devcontainer_rc +fi + diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..5dd8f25e5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +# avoid blowing the cache over build files + +.dockerignore + +external +*build* +*prefix* +*.pyc +tools/python/venv +venv* +tools/python/.coverage +*.egg-info + + +tools/python/build +tools/python/dist + +docs/build/ \ No newline at end of file diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml new file mode 100644 index 000000000..60472bbc8 --- /dev/null +++ b/.github/workflows/container.yml @@ -0,0 +1,99 @@ +name: Container CI + +on: + push: + # Build for main, tags and PRs + # Builds from PRs are not pushed to the registry because of rules in the steps + branches: + # push these changes to the branch 'docker' for container publishing + - docker + tags: + - "*" + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 # All history + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Log in to GitHub Docker Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=raw,value=latest + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Docker meta config build + id: meta-build + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository }}-developer + tags: | + type=ref,event=branch + type=ref,event=tag + type=raw,value=latest + + - name: Docker build build image + uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile + target: developer + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta-build.outputs.tags }} + labels: ${{ steps.meta-build.outputs.labels }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + + - name: Docker meta config runtime + id: meta-runtime + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository }}-runtime + tags: | + type=ref,event=branch + type=ref,event=tag + type=raw,value=latest + + - name: Docker build runtime image + uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile + target: runtime + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta-runtime.outputs.tags }} + labels: ${{ steps.meta-runtime.outputs.labels }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.gitignore b/.gitignore index 3a6b749fb..0c14247ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /external /*build* /*prefix/ +/vscode_prefix *.pyc /tools/python/venv/ /.settings @@ -15,7 +16,6 @@ tools/python/.coverage .idea/ cmake-build*/ -.vscode/ tools/imagej/plugins/* !tools/imagej/plugins/Odin-Data/ diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 000000000..989aa1c29 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,9 @@ +{ + "recommendations": [ + "ms-vscode-remote.remote-containers", + "twxs.cmake", + "ms-vscode.cpptools", + "ms-python.python", + "ms-python.vscode-pylance", + ] +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..3a1731d61 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "MetaWriter", + "type": "python", + "request": "launch", + "module": "odin_data.meta_writer.meta_writer_app", + "args": [ + "--log-level=DEBUG" + ] + }, + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..baf2361ba --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,20 @@ +{ + "cmake.generator": "Unix Makefiles", + "cmake.sourceDirectory": "${workspaceFolder}/cpp", + "cmake.buildDirectory": "${workspaceFolder}/vscode_build", + "cmake.installPrefix": "${workspaceFolder}/vscode_prefix", + "cmake.configureArgs": [ + // all dependencies from system inside the container + ], + "C_Cpp.default.configurationProvider": "ms-vscode.cmake-tools", + "editor.formatOnSave": true, + "[python]": { + "editor.codeActionsOnSave": { + "source.organizeImports": "explicit" + }, + "editor.defaultFormatter": "ms-python.python", + "editor.rulers": [ + 88 + ] + } +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 000000000..debeb157f --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,18 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "Run Dummy Decoder System Test", + "type": "shell", + "options": { + "env": { + "INSTALL_PREFIX": "${workspaceFolder}/vscode_prefix" + }, + }, + "command": "${workspaceFolder}/vscode_prefix/bin/odinDataTest --json=${workspaceFolder}/vscode_prefix/test_config/dummyUDP.json", + "problemMatcher": [], + } + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..096a26261 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,70 @@ +# shared setup stage ########################################################### +FROM ubuntu:24.04 AS common + +ENV PATH=/odin/bin:/venv/bin:$PATH + +# get fundamental packages +RUN apt-get update -y && \ + apt-get install --no-install-recommends -y curl \ + tar ca-certificates software-properties-common && \ + apt-get -y clean all +# get zellij +RUN curl -L https://github.com/zellij-org/zellij/releases/download/v0.40.1/zellij-x86_64-unknown-linux-musl.tar.gz -o zellij.tar.gz && \ + tar -xvf zellij.tar.gz -C /usr/bin && \ + rm zellij.tar.gz +# setup zellij +RUN mkdir -p ~/.config/zellij && \ + zellij setup --dump-config > ~/.config/zellij/config.kdl + +# developer stage for devcontainer ############################################# +FROM common AS developer + +# system depenedencies +RUN add-apt-repository -y ppa:deadsnakes/ppa && \ + apt-get update -y && apt-get install -y --no-install-recommends \ + # General build + build-essential cmake git \ + # odin-data C++ dependencies + libblosc-dev libboost-all-dev libhdf5-dev liblog4cxx-dev libpcap-dev libczmq-dev \ + # python + python3.11-dev python3.11-venv && \ + # tidy up + apt-get -y clean all + +# python dependencies +RUN python3.11 -m ensurepip && \ + python3.11 -m venv /venv && \ + python -m pip install --upgrade pip && \ + python -m pip install git+https://github.com/odin-detector/odin-control@1.3.0 + +# build stage - throwaway stage for runtime assets ############################# +FROM developer AS build + +# fetch the source +WORKDIR /tmp/odin-data +COPY . . + +# C++ +RUN mkdir /odin && \ + mkdir -p build && cd build && \ + cmake -DCMAKE_INSTALL_PREFIX=/odin ../cpp && \ + make -j8 VERBOSE=1 && \ + make install + +# Python +RUN python -m pip install /tmp/odin-data/python[meta_writer] + +# Runtime stage ################################################################ +FROM common as runtime + +# runtime system dependencies +RUN apt-get update -y && apt-get install -y --no-install-recommends \ + # odin-data C++ dependencies + libblosc-dev libboost-all-dev libhdf5-dev liblog4cxx-dev libpcap-dev libczmq-dev && \ + # tidy up + apt-get -y clean all + +COPY --from=build /odin /odin +COPY --from=build /venv /venv + +WORKDIR /odin