From 0457c9a8f664ce7197c597dca7a3e152e3198697 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 11:02:27 +0100 Subject: [PATCH 01/94] add github workflow --- .github/workflows/docker.yml | 51 ++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000000..4dafb8cb5fe --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,51 @@ +name: Docker + +on: [push, pull_request] + +jobs: + + create-carla-prerequisites-image: + name: create carla prerequisites image + runs-on: self-hosted + steps: + - uses: docker/login-action@v3 + name: Login to Docker Hub + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - uses: docker/build-push-action@v5 + name: Build and push + with: + file: Util/Docker/Prerequisites.Dockerfile + tags: cgellerac/carla-prerequisites + outputs: type=docker,dest=/tmp/carla-prerequisites.tar + - uses: actions/upload-artifact@v3 + name: Upload artifact + with: + name: carla-prerequisites + path: /tmp/carla-prerequisites.tar + + create-carla-source-image: + name: create carla source image + runs-on: self-hosted + steps: + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: carla-prerequisites + path: /tmp + - name: Load image + run: | + docker load --input /tmp/carla-prerequisites.tar + docker image ls -a + - uses: docker/login-action@v3 + name: Login to Docker Hub + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - uses: docker/build-push-action@v5 + name: Build and push + with: + push: true + file: Util/Docker/Carla.Dockerfile + tags: cgellerac/carla-source \ No newline at end of file From 14a3dc995381dcdc8f8f83655a781a27bc5de5e5 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 11:11:28 +0100 Subject: [PATCH 02/94] add epic user secrets --- .github/workflows/docker.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4dafb8cb5fe..20347ba3191 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,7 +17,10 @@ jobs: name: Build and push with: file: Util/Docker/Prerequisites.Dockerfile - tags: cgellerac/carla-prerequisites + build-args: + EPIC_USER=${{ secrets.EPIC_USER }} + EPIC_PASS=${{ secrets.EPIC_PASS }} + tags: carla-prerequisites outputs: type=docker,dest=/tmp/carla-prerequisites.tar - uses: actions/upload-artifact@v3 name: Upload artifact @@ -48,4 +51,6 @@ jobs: with: push: true file: Util/Docker/Carla.Dockerfile + build-args: + GIT_PASS= tags: cgellerac/carla-source \ No newline at end of file From e31841c2a5da51a3e9cb4c66d749bf5591d43e0c Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 11:32:31 +0100 Subject: [PATCH 03/94] add setup buildx action --- .github/workflows/docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 20347ba3191..131442ba9c5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -8,6 +8,8 @@ jobs: name: create carla prerequisites image runs-on: self-hosted steps: + - uses: docker/setup-buildx-action@v3 + name: Set up Docker Buildx - uses: docker/login-action@v3 name: Login to Docker Hub with: From 1f8364b81323057f748ef719e14414f7668a1579 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 12:36:38 +0100 Subject: [PATCH 04/94] update Dockerfile --- Util/Docker/Prerequisites.Dockerfile | 117 ++++++++++++++++++++++----- 1 file changed, 98 insertions(+), 19 deletions(-) diff --git a/Util/Docker/Prerequisites.Dockerfile b/Util/Docker/Prerequisites.Dockerfile index 7cd78116707..d2c1f5c8939 100644 --- a/Util/Docker/Prerequisites.Dockerfile +++ b/Util/Docker/Prerequisites.Dockerfile @@ -1,28 +1,72 @@ -FROM ubuntu:18.04 +FROM nvidia/cuda:11.3.1-runtime-ubuntu20.04 +ARG EPIC_USER +ARG EPIC_PASS +ENV DEBIAN_FRONTEND=noninteractive + +RUN echo $EPIC_USER +RUN echo $EPIC_PASS + +# Setup USER root -ARG EPIC_USER=user -ARG EPIC_PASS=pass -ENV DEBIAN_FRONTEND=noninteractive +# Install GnuPG2 and import key +RUN apt-get update && apt-get install -y gnupg2 +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys A4B469963BF863CC + +# Update apt-get RUN apt-get update ; \ apt-get install -y wget software-properties-common && \ add-apt-repository ppa:ubuntu-toolchain-r/test && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|apt-key add - && \ - apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" && \ - apt-get update ; \ + add-apt-repository ppa:deadsnakes/ppa + +# Install Dependencies +RUN apt-get update ; \ apt-get install -y build-essential \ + xdg-user-dirs \ + sudo \ + libasound2 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libcairo2 \ + libfontconfig1 \ + libfreetype6 \ + libglu1 \ + libglvnd-dev \ + libnss3 \ + libnspr4 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libsm6 \ + libxcomposite1 \ + libxcursor1 \ + libxdamage1 \ + libxi6 \ + libxrandr2 \ + libxrender1 \ + libxss1 \ + libxtst6 \ + libxv1 \ + x11-xkb-utils \ + xauth \ + xfonts-base \ + xkb-data \ clang-8 \ lld-8 \ g++-7 \ cmake \ ninja-build \ libvulkan1 \ + libvulkan-dev \ + vulkan-tools \ python \ - python-pip \ + python3.8 \ + python3.10 \ python-dev \ - python3-dev \ - python3-pip \ + python3.8-dev \ + python3.10-dev \ + python3.8-distutils \ + python3.10-distutils \ libpng-dev \ libtiff5-dev \ libjpeg-dev \ @@ -35,17 +79,52 @@ RUN apt-get update ; \ rsync \ libxml2-dev \ git \ - aria2 && \ - pip3 install -Iv setuptools==47.3.1 && \ - pip3 install distro && \ + nano \ + aria2 + +# Install pip for python 3.8 and 3.10 +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3.8 get-pip.py && \ + python3.10 get-pip.py && \ + rm get-pip.py + +# Install Python Packages +RUN pip3.8 install -Iv setuptools==47.3.1 && \ + pip3.8 install distro && \ + pip3.8 install pygame && \ + pip3.8 install numpy + +RUN pip3.10 install -Iv setuptools==47.3.1 && \ + pip3.10 install distro && \ + pip3.10 install pygame && \ + pip3.10 install numpy + +# Set Alternatives and Defaults +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 && \ + update-alternatives --set python3 /usr/bin/python3.10 && \ update-alternatives --install /usr/bin/clang++ clang++ /usr/lib/llvm-8/bin/clang++ 180 && \ update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-8/bin/clang 180 -RUN useradd -m carla -COPY --chown=carla:carla . /home/carla -USER carla -WORKDIR /home/carla -ENV UE4_ROOT /home/carla/UE4.26 +# Enable Vulkan support for NVIDIA GPUs +RUN apt-get update && apt-get install -y --no-install-recommends libvulkan1 && \ + rm -rf /var/lib/apt/lists/* && \ + VULKAN_API_VERSION=`dpkg -s libvulkan1 | grep -oP 'Version: [0-9|\.]+' | grep -oP '[0-9|\.]+'` && \ + mkdir -p /etc/vulkan/icd.d/ && \ + echo \ + "{\ + \"file_format_version\" : \"1.0.0\",\ + \"ICD\": {\ + \"library_path\": \"libGLX_nvidia.so.0\",\ + \"api_version\" : \"${VULKAN_API_VERSION}\"\ + }\ + }" > /etc/vulkan/icd.d/nvidia_icd.json + +# Setup carlauser and UnrealEngine +RUN useradd -m carlauser && echo "carlauser:carlauser" | chpasswd && adduser carlauser sudo +COPY --chown=carlauser:carlauser . /home/carlauser +USER carlauser +WORKDIR /home/carlauser +ENV UE4_ROOT /home/carlauser/UE4.26 RUN git clone --depth 1 -b carla "https://${EPIC_USER}:${EPIC_PASS}@github.com/CarlaUnreal/UnrealEngine.git" ${UE4_ROOT} @@ -54,4 +133,4 @@ RUN cd $UE4_ROOT && \ ./GenerateProjectFiles.sh && \ make -WORKDIR /home/carla/ +WORKDIR /home/carlauser/ From 7f75890956a2bbc3744795436dcda2bc66dc7728 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 13:07:59 +0100 Subject: [PATCH 05/94] update base image --- .github/workflows/docker.yml | 3 - Util/Docker/Prerequisites.Dockerfile | 101 +++++++-------------------- 2 files changed, 26 insertions(+), 78 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 131442ba9c5..ba66ef9eafe 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,9 +19,6 @@ jobs: name: Build and push with: file: Util/Docker/Prerequisites.Dockerfile - build-args: - EPIC_USER=${{ secrets.EPIC_USER }} - EPIC_PASS=${{ secrets.EPIC_PASS }} tags: carla-prerequisites outputs: type=docker,dest=/tmp/carla-prerequisites.tar - uses: actions/upload-artifact@v3 diff --git a/Util/Docker/Prerequisites.Dockerfile b/Util/Docker/Prerequisites.Dockerfile index d2c1f5c8939..a1c60dfbbb8 100644 --- a/Util/Docker/Prerequisites.Dockerfile +++ b/Util/Docker/Prerequisites.Dockerfile @@ -1,56 +1,24 @@ -FROM nvidia/cuda:11.3.1-runtime-ubuntu20.04 +FROM nvidia/cuda:12.3.1-runtime-ubuntu22.04 -ARG EPIC_USER -ARG EPIC_PASS +USER root + +ARG EPIC_USER=christiangeller +ARG EPIC_PASS=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 ENV DEBIAN_FRONTEND=noninteractive +# TODO: DELETE RUN echo $EPIC_USER RUN echo $EPIC_PASS -# Setup -USER root - -# Install GnuPG2 and import key -RUN apt-get update && apt-get install -y gnupg2 -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys A4B469963BF863CC - -# Update apt-get +# Install dependencies RUN apt-get update ; \ apt-get install -y wget software-properties-common && \ add-apt-repository ppa:ubuntu-toolchain-r/test && \ - add-apt-repository ppa:deadsnakes/ppa - -# Install Dependencies -RUN apt-get update ; \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|apt-key add - && \ + apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" && \ + apt-get update ; \ apt-get install -y build-essential \ - xdg-user-dirs \ - sudo \ - libasound2 \ - libatk1.0-0 \ - libatk-bridge2.0-0 \ - libcairo2 \ - libfontconfig1 \ - libfreetype6 \ - libglu1 \ libglvnd-dev \ - libnss3 \ - libnspr4 \ - libpango-1.0-0 \ - libpangocairo-1.0-0 \ - libsm6 \ - libxcomposite1 \ - libxcursor1 \ - libxdamage1 \ - libxi6 \ - libxrandr2 \ - libxrender1 \ - libxss1 \ - libxtst6 \ - libxv1 \ - x11-xkb-utils \ - xauth \ - xfonts-base \ - xkb-data \ clang-8 \ lld-8 \ g++-7 \ @@ -59,14 +27,10 @@ RUN apt-get update ; \ libvulkan1 \ libvulkan-dev \ vulkan-tools \ - python \ - python3.8 \ - python3.10 \ - python-dev \ - python3.8-dev \ - python3.10-dev \ - python3.8-distutils \ - python3.10-distutils \ + python3 \ + python3-dev \ + python3-pip \ + python3-distutils \ libpng-dev \ libtiff5-dev \ libjpeg-dev \ @@ -82,27 +46,14 @@ RUN apt-get update ; \ nano \ aria2 -# Install pip for python 3.8 and 3.10 -RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python3.8 get-pip.py && \ - python3.10 get-pip.py && \ - rm get-pip.py - # Install Python Packages -RUN pip3.8 install -Iv setuptools==47.3.1 && \ - pip3.8 install distro && \ - pip3.8 install pygame && \ - pip3.8 install numpy - -RUN pip3.10 install -Iv setuptools==47.3.1 && \ - pip3.10 install distro && \ - pip3.10 install pygame && \ - pip3.10 install numpy +RUN pip3 install -Iv setuptools==47.3.1 && \ + pip3 install distro && \ + pip3 install pygame && \ + pip3 install numpy # Set Alternatives and Defaults -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 && \ - update-alternatives --set python3 /usr/bin/python3.10 && \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/lib/llvm-8/bin/clang++ 180 && \ +RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/lib/llvm-8/bin/clang++ 180 && \ update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-8/bin/clang 180 # Enable Vulkan support for NVIDIA GPUs @@ -119,12 +70,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends libvulkan1 && \ }\ }" > /etc/vulkan/icd.d/nvidia_icd.json -# Setup carlauser and UnrealEngine -RUN useradd -m carlauser && echo "carlauser:carlauser" | chpasswd && adduser carlauser sudo -COPY --chown=carlauser:carlauser . /home/carlauser -USER carlauser -WORKDIR /home/carlauser -ENV UE4_ROOT /home/carlauser/UE4.26 +# Setup carla and UnrealEngine +RUN useradd -m carla && echo "carla:carla" | chpasswd && adduser carla sudo +COPY --chown=carla:carla . /home/carla +USER carla +WORKDIR /home/carla +ENV UE4_ROOT /home/carla/UE4.26 RUN git clone --depth 1 -b carla "https://${EPIC_USER}:${EPIC_PASS}@github.com/CarlaUnreal/UnrealEngine.git" ${UE4_ROOT} @@ -133,4 +84,4 @@ RUN cd $UE4_ROOT && \ ./GenerateProjectFiles.sh && \ make -WORKDIR /home/carlauser/ +WORKDIR /home/carla/ From 3eaa3ed4b98eb61225f62e1a38eaee0c25fea3b7 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 13:21:52 +0100 Subject: [PATCH 06/94] change ubuntu to 20.04 --- Util/Docker/Prerequisites.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Util/Docker/Prerequisites.Dockerfile b/Util/Docker/Prerequisites.Dockerfile index a1c60dfbbb8..f41a92d0e4a 100644 --- a/Util/Docker/Prerequisites.Dockerfile +++ b/Util/Docker/Prerequisites.Dockerfile @@ -1,4 +1,4 @@ -FROM nvidia/cuda:12.3.1-runtime-ubuntu22.04 +FROM nvidia/cuda:11.3.1-runtime-ubuntu20.04 # TODO: Change to 22.04 USER root From 63a857c01b2d61627736bb3c6a25fd425014fcfe Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 13:27:13 +0100 Subject: [PATCH 07/94] update Dockerfile --- Util/Docker/Prerequisites.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Util/Docker/Prerequisites.Dockerfile b/Util/Docker/Prerequisites.Dockerfile index f41a92d0e4a..37015425c8c 100644 --- a/Util/Docker/Prerequisites.Dockerfile +++ b/Util/Docker/Prerequisites.Dockerfile @@ -1,4 +1,5 @@ -FROM nvidia/cuda:11.3.1-runtime-ubuntu20.04 # TODO: Change to 22.04 +FROM nvidia/cuda:11.3.1-runtime-ubuntu20.04 +# TODO: Change to 22.04 USER root From 3a3414b598e9fbf29af15a10c4df7a0624911835 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 13:34:01 +0100 Subject: [PATCH 08/94] install python correctly --- Util/Docker/Prerequisites.Dockerfile | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/Util/Docker/Prerequisites.Dockerfile b/Util/Docker/Prerequisites.Dockerfile index 37015425c8c..369ab7d1164 100644 --- a/Util/Docker/Prerequisites.Dockerfile +++ b/Util/Docker/Prerequisites.Dockerfile @@ -15,8 +15,7 @@ RUN echo $EPIC_PASS RUN apt-get update ; \ apt-get install -y wget software-properties-common && \ add-apt-repository ppa:ubuntu-toolchain-r/test && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|apt-key add - && \ - apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" && \ + add-apt-repository ppa:deadsnakes/ppa && \ # add newer Python version apt-get update ; \ apt-get install -y build-essential \ libglvnd-dev \ @@ -28,10 +27,10 @@ RUN apt-get update ; \ libvulkan1 \ libvulkan-dev \ vulkan-tools \ - python3 \ - python3-dev \ - python3-pip \ - python3-distutils \ + python3.10 \ + python3.10-dev \ + python3.10-pip \ + python3.10-distutils \ libpng-dev \ libtiff5-dev \ libjpeg-dev \ @@ -48,13 +47,15 @@ RUN apt-get update ; \ aria2 # Install Python Packages -RUN pip3 install -Iv setuptools==47.3.1 && \ - pip3 install distro && \ - pip3 install pygame && \ - pip3 install numpy +RUN pip3.10 install -Iv setuptools==47.3.1 && \ + pip3.10 install distro && \ + pip3.10 install pygame && \ + pip3.10 install numpy # Set Alternatives and Defaults -RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/lib/llvm-8/bin/clang++ 180 && \ +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 && \ + update-alternatives --set python3 /usr/bin/python3.10 && \ +update-alternatives --install /usr/bin/clang++ clang++ /usr/lib/llvm-8/bin/clang++ 180 && \ update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-8/bin/clang 180 # Enable Vulkan support for NVIDIA GPUs From 280935165af15021918925cd168b1164e2efa398 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 13:37:39 +0100 Subject: [PATCH 09/94] fix Dockerfile --- Util/Docker/Prerequisites.Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Util/Docker/Prerequisites.Dockerfile b/Util/Docker/Prerequisites.Dockerfile index 369ab7d1164..0bc8ca5b286 100644 --- a/Util/Docker/Prerequisites.Dockerfile +++ b/Util/Docker/Prerequisites.Dockerfile @@ -15,10 +15,9 @@ RUN echo $EPIC_PASS RUN apt-get update ; \ apt-get install -y wget software-properties-common && \ add-apt-repository ppa:ubuntu-toolchain-r/test && \ - add-apt-repository ppa:deadsnakes/ppa && \ # add newer Python version + add-apt-repository ppa:deadsnakes/ppa && \ apt-get update ; \ apt-get install -y build-essential \ - libglvnd-dev \ clang-8 \ lld-8 \ g++-7 \ @@ -27,6 +26,7 @@ RUN apt-get update ; \ libvulkan1 \ libvulkan-dev \ vulkan-tools \ + libglvnd-dev \ python3.10 \ python3.10-dev \ python3.10-pip \ @@ -43,7 +43,6 @@ RUN apt-get update ; \ rsync \ libxml2-dev \ git \ - nano \ aria2 # Install Python Packages From 3ba1888eefd37ad16557096becc5bddc3c857ead Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 13:45:42 +0100 Subject: [PATCH 10/94] custom pip3.10 install --- Util/Docker/Prerequisites.Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Util/Docker/Prerequisites.Dockerfile b/Util/Docker/Prerequisites.Dockerfile index 0bc8ca5b286..8c1cd52c444 100644 --- a/Util/Docker/Prerequisites.Dockerfile +++ b/Util/Docker/Prerequisites.Dockerfile @@ -45,6 +45,11 @@ RUN apt-get update ; \ git \ aria2 +# Install pip for python 3.10 +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3.10 get-pip.py && \ + rm get-pip.py + # Install Python Packages RUN pip3.10 install -Iv setuptools==47.3.1 && \ pip3.10 install distro && \ From 1a3b6ef0937f0ca49a8bc6180b8336d688265e9a Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 13:45:53 +0100 Subject: [PATCH 11/94] custom pip3.10 install --- Util/Docker/Prerequisites.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Util/Docker/Prerequisites.Dockerfile b/Util/Docker/Prerequisites.Dockerfile index 8c1cd52c444..a23a4ce3f45 100644 --- a/Util/Docker/Prerequisites.Dockerfile +++ b/Util/Docker/Prerequisites.Dockerfile @@ -29,7 +29,6 @@ RUN apt-get update ; \ libglvnd-dev \ python3.10 \ python3.10-dev \ - python3.10-pip \ python3.10-distutils \ libpng-dev \ libtiff5-dev \ From 9b20310bde77122e36e924d46e81eba4a9ea2460 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 14:17:32 +0100 Subject: [PATCH 12/94] use secrets --- .github/workflows/docker.yml | 2 ++ Util/Docker/Prerequisites.Dockerfile | 8 ++------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ba66ef9eafe..034d3638635 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,6 +19,8 @@ jobs: name: Build and push with: file: Util/Docker/Prerequisites.Dockerfile + EPIC_USER=${{ secrets.EPIC_USER }} + EPIC_PASS=${{ secrets.EPIC_PASS }} tags: carla-prerequisites outputs: type=docker,dest=/tmp/carla-prerequisites.tar - uses: actions/upload-artifact@v3 diff --git a/Util/Docker/Prerequisites.Dockerfile b/Util/Docker/Prerequisites.Dockerfile index a23a4ce3f45..98154c8f9b1 100644 --- a/Util/Docker/Prerequisites.Dockerfile +++ b/Util/Docker/Prerequisites.Dockerfile @@ -3,14 +3,10 @@ FROM nvidia/cuda:11.3.1-runtime-ubuntu20.04 USER root -ARG EPIC_USER=christiangeller -ARG EPIC_PASS=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 +ARG EPIC_USER +ARG EPIC_PASS ENV DEBIAN_FRONTEND=noninteractive -# TODO: DELETE -RUN echo $EPIC_USER -RUN echo $EPIC_PASS - # Install dependencies RUN apt-get update ; \ apt-get install -y wget software-properties-common && \ From 8dcda534ddc89618200a05966c6745308af3f0be Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 14:29:36 +0100 Subject: [PATCH 13/94] update Carla.Dockerfile --- Util/Docker/Carla.Dockerfile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Util/Docker/Carla.Dockerfile b/Util/Docker/Carla.Dockerfile index 65f5d67dd5d..e474efbdec7 100644 --- a/Util/Docker/Carla.Dockerfile +++ b/Util/Docker/Carla.Dockerfile @@ -5,15 +5,17 @@ ARG GIT_BRANCH USER carla WORKDIR /home/carla -RUN cd /home/carla/ && \ - if [ -z ${GIT_BRANCH+x} ]; then git clone --depth 1 https://github.com/carla-simulator/carla.git; \ - else git clone --depth 1 --branch $GIT_BRANCH https://github.com/carla-simulator/carla.git; fi && \ - cd /home/carla/carla && \ +COPY . /home/carla/carla + +RUN cd /home/carla/carla && \ ./Update.sh && \ make CarlaUE4Editor && \ - make PythonAPI && \ + make PythonAPI ARGS="--python-version='3.10'" && \ make build.utils && \ - make package && \ - rm -r /home/carla/carla/Dist + make package WORKDIR /home/carla/carla + +COPY entrypoint.sh /usr/bin/ +ENTRYPOINT ["/usr/bin/entrypoint.sh"] +CMD [""] \ No newline at end of file From dc7179205d2f3ee6ac0f2bb3453182c66f6b3e40 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 14:36:17 +0100 Subject: [PATCH 14/94] add build-args --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 034d3638635..35f70064aaa 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,6 +19,7 @@ jobs: name: Build and push with: file: Util/Docker/Prerequisites.Dockerfile + build-args: EPIC_USER=${{ secrets.EPIC_USER }} EPIC_PASS=${{ secrets.EPIC_PASS }} tags: carla-prerequisites From f7c85be59ed1e9fd28a80a7c0a763b96120de955 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 14:43:19 +0100 Subject: [PATCH 15/94] apply changes to BuildUtilsDocker --- Util/BuildTools/BuildUtilsDocker.sh | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Util/BuildTools/BuildUtilsDocker.sh b/Util/BuildTools/BuildUtilsDocker.sh index 2c85a7b053c..609d2b5ff42 100755 --- a/Util/BuildTools/BuildUtilsDocker.sh +++ b/Util/BuildTools/BuildUtilsDocker.sh @@ -12,22 +12,18 @@ if [ -f "${FBX2OBJ_DIST}/FBX2OBJ" ]; then exit fi -LIB_NAME=fbx202001_fbxsdk_linux -FBXSDK_URL=https://www.autodesk.com/content/dam/autodesk/www/adn/fbx/2020-0-1/${LIB_NAME}.tar.gz +LIB_NAME=fbx202034_fbxsdk_linux +FBXSDK_URL=https://rwth-aachen.sciebo.de/s/z4KgTOD39v4I8Qw/download # Private Sciebo if [ ! -d "${FBX2OBJ_DEP_FOLDER}" ]; then log "Downloading FBX SDK..." - wget -c "${FBXSDK_URL}" -P "${CARLA_DOCKER_UTILS_FOLDER}" --user-agent="Mozilla" + wget -O "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}.zip" "${FBXSDK_URL}" echo "Unpacking..." mkdir -p "${FBX2OBJ_DEP_FOLDER}" - tar -xvzf "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}.tar.gz" -C "${CARLA_DOCKER_UTILS_FOLDER}" "${LIB_NAME}" + unzip "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}.zip" -d "${FBX2OBJ_DEP_FOLDER}" rm "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}.tar.gz" - echo "Installing FBX SDK..." - echo -e "y\nyes\nn\n" | "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}" "${FBX2OBJ_DEP_FOLDER}" - echo - rm "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}" fi log "Compiling FBX2OBJ..." From 50bd1ce5dcee01b1ba71d7fb06b51ed066736db5 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 14:47:30 +0100 Subject: [PATCH 16/94] fix epic secret handling --- .github/workflows/docker.yml | 2 +- Util/Docker/Prerequisites.Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 35f70064aaa..7f04b56f7ce 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,7 +19,7 @@ jobs: name: Build and push with: file: Util/Docker/Prerequisites.Dockerfile - build-args: + build-args: | EPIC_USER=${{ secrets.EPIC_USER }} EPIC_PASS=${{ secrets.EPIC_PASS }} tags: carla-prerequisites diff --git a/Util/Docker/Prerequisites.Dockerfile b/Util/Docker/Prerequisites.Dockerfile index 98154c8f9b1..8d9e4da4b3b 100644 --- a/Util/Docker/Prerequisites.Dockerfile +++ b/Util/Docker/Prerequisites.Dockerfile @@ -3,8 +3,8 @@ FROM nvidia/cuda:11.3.1-runtime-ubuntu20.04 USER root -ARG EPIC_USER -ARG EPIC_PASS +ARG EPIC_USER=user +ARG EPIC_PASS=pass ENV DEBIAN_FRONTEND=noninteractive # Install dependencies From bed8e9d45343b9f33570547a68f6450ad8a06ea7 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 15:09:45 +0100 Subject: [PATCH 17/94] add carla packaged image job --- .github/workflows/docker.yml | 39 +++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7f04b56f7ce..53bb68c63b7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -53,6 +53,39 @@ jobs: with: push: true file: Util/Docker/Carla.Dockerfile - build-args: - GIT_PASS= - tags: cgellerac/carla-source \ No newline at end of file + tags: cgellerac/carla-source + - uses: shrink/actions-docker-extract@v3 + name: extract carla package + with: + image: cgellerac/carla-source + output: /home/carla/Dist/*.tar.gz + destination: /tmp/carla-package.tar.gz + - uses: actions/upload-artifact@v3 + name: Upload artifact + with: + name: carla-package + path: /tmp/carla-package.tar.gz + + create-carla-image: + name: create carla image + runs-on: self-hosted + steps: + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: carla-package + path: /tmp + - name: Extract artifact + run: tar -xvzf /tmp/carla-package.tar.gz -C /build + - uses: docker/login-action@v3 + name: Login to Docker Hub + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - uses: docker/build-push-action@v5 + name: Build and push + with: + push: true + file: /build/Dockerfile + context: /build + tags: cgellerac/carla \ No newline at end of file From ff8c1252e8a13396977500210e2a675127d7a173 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 15:10:14 +0100 Subject: [PATCH 18/94] add todo --- Util/Docker/Release.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Util/Docker/Release.Dockerfile b/Util/Docker/Release.Dockerfile index 39d4b6d47a9..3cb65be0469 100644 --- a/Util/Docker/Release.Dockerfile +++ b/Util/Docker/Release.Dockerfile @@ -3,7 +3,8 @@ # # sudo -E docker run --rm --gpus all -it --net=host carla:latest /bin/bash -FROM nvidia/vulkan:1.1.121-cuda-10.1--ubuntu18.04 +FROM nvidia/vulkan:1.1.121-cuda-10.1--ubuntu18.04 +# TODO: Change to 22.04 RUN apt-key adv --fetch-keys "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub" From 2a099eb393f4e2d608a0b5cfed213b9b899a0ad1 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 12 Dec 2023 15:21:46 +0100 Subject: [PATCH 19/94] add release PythonAPI actions --- .github/workflows/docker.yml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 53bb68c63b7..dfdd3bf0e8d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -60,11 +60,28 @@ jobs: image: cgellerac/carla-source output: /home/carla/Dist/*.tar.gz destination: /tmp/carla-package.tar.gz + - uses: shrink/actions-docker-extract@v3 + name: extract PythonAPI package + with: + image: cgellerac/carla-source + output: /home/carla/PythonAPI + destination: /tmp/carla-python-api - uses: actions/upload-artifact@v3 - name: Upload artifact + name: Upload carla-package artifact with: name: carla-package path: /tmp/carla-package.tar.gz + - uses: actions/upload-artifact@v3 + name: Upload carla-python-api artifact + with: + name: carla-python-api + path: /tmp/carla-python-api + - uses: actions/upload-release-asset@v1 + name: Upload Release Asset + id: upload-release-asset + with: + asset_path: /tmp/carla-python-api + asset_name: carla-python-api create-carla-image: name: create carla image @@ -88,4 +105,5 @@ jobs: push: true file: /build/Dockerfile context: /build - tags: cgellerac/carla \ No newline at end of file + tags: cgellerac/carla + \ No newline at end of file From a0ac13d6a0efcb9a5c001b942323a3dcf11a4583 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Wed, 13 Dec 2023 11:28:21 +0100 Subject: [PATCH 20/94] update workflow --- .github/workflows/docker.yml | 65 +++++++++++++++++------------------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index dfdd3bf0e8d..ac083263866 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -10,11 +10,6 @@ jobs: steps: - uses: docker/setup-buildx-action@v3 name: Set up Docker Buildx - - uses: docker/login-action@v3 - name: Login to Docker Hub - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - uses: docker/build-push-action@v5 name: Build and push with: @@ -23,26 +18,12 @@ jobs: EPIC_USER=${{ secrets.EPIC_USER }} EPIC_PASS=${{ secrets.EPIC_PASS }} tags: carla-prerequisites - outputs: type=docker,dest=/tmp/carla-prerequisites.tar - - uses: actions/upload-artifact@v3 - name: Upload artifact - with: - name: carla-prerequisites - path: /tmp/carla-prerequisites.tar create-carla-source-image: name: create carla source image runs-on: self-hosted + needs: create-carla-prerequisites-image steps: - - name: Download artifact - uses: actions/download-artifact@v3 - with: - name: carla-prerequisites - path: /tmp - - name: Load image - run: | - docker load --input /tmp/carla-prerequisites.tar - docker image ls -a - uses: docker/login-action@v3 name: Login to Docker Hub with: @@ -56,49 +37,63 @@ jobs: tags: cgellerac/carla-source - uses: shrink/actions-docker-extract@v3 name: extract carla package + id: extract_carla_package with: image: cgellerac/carla-source output: /home/carla/Dist/*.tar.gz - destination: /tmp/carla-package.tar.gz + destination: /tmp/ + - uses: actions/upload-artifact@v3 + name: Upload carla-package artifact + with: + name: carla-package + path: ${{ steps.extract_carla_package.outputs.destination }} - uses: shrink/actions-docker-extract@v3 name: extract PythonAPI package + id: extract_python_api with: image: cgellerac/carla-source output: /home/carla/PythonAPI - destination: /tmp/carla-python-api - - uses: actions/upload-artifact@v3 - name: Upload carla-package artifact - with: - name: carla-package - path: /tmp/carla-package.tar.gz + destination: /tmp/ - uses: actions/upload-artifact@v3 name: Upload carla-python-api artifact with: name: carla-python-api - path: /tmp/carla-python-api + path: ${{ steps.extract_python_api.outputs.destination }} + - uses: actions/create-release@v1 + name: Create Release + id: create_release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + draft: false + prerelease: false - uses: actions/upload-release-asset@v1 name: Upload Release Asset id: upload-release-asset with: - asset_path: /tmp/carla-python-api + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ${{ steps.extract_python_api.outputs.destination }} asset_name: carla-python-api create-carla-image: name: create carla image runs-on: self-hosted + needs: create-carla-source-image steps: + - uses: docker/login-action@v3 + name: Login to Docker Hub + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} - name: Download artifact uses: actions/download-artifact@v3 with: name: carla-package path: /tmp - name: Extract artifact - run: tar -xvzf /tmp/carla-package.tar.gz -C /build - - uses: docker/login-action@v3 - name: Login to Docker Hub - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + run: tar -xvzf /tmp/carla-package.tar.gz -C /build - uses: docker/build-push-action@v5 name: Build and push with: From 6ed4673ac5b2c00326b21cdc07b31560e137a6ad Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Wed, 13 Dec 2023 16:02:13 +0100 Subject: [PATCH 21/94] Beautify workflow --- .github/workflows/docker.yml | 84 +++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 35 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ac083263866..06742c73cab 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,101 +4,115 @@ on: [push, pull_request] jobs: + # create carla-prerequisites image create-carla-prerequisites-image: - name: create carla prerequisites image + name: Create carla-prerequisites image runs-on: self-hosted steps: - - uses: docker/setup-buildx-action@v3 - name: Set up Docker Buildx + - uses: docker/build-push-action@v5 - name: Build and push + name: Build image with: file: Util/Docker/Prerequisites.Dockerfile + tags: carla-prerequisites build-args: | EPIC_USER=${{ secrets.EPIC_USER }} EPIC_PASS=${{ secrets.EPIC_PASS }} - tags: carla-prerequisites + # create and push carla-source image create-carla-source-image: - name: create carla source image + name: Create carla-source image runs-on: self-hosted needs: create-carla-prerequisites-image steps: + - uses: docker/login-action@v3 name: Login to Docker Hub with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - uses: docker/build-push-action@v5 - name: Build and push + name: Build and push image with: - push: true file: Util/Docker/Carla.Dockerfile tags: cgellerac/carla-source + push: true + + # provide artifacts and releases + provide-artifacts: + name: Provide carla artifacts + runs-on: self-hosted + needs: create-carla-source-image + steps: + + # provide carla-package - uses: shrink/actions-docker-extract@v3 - name: extract carla package + name: Extract carla-package id: extract_carla_package with: image: cgellerac/carla-source - output: /home/carla/Dist/*.tar.gz + path: /home/carla/Dist/*.tar.gz # TODO: check if valid destination: /tmp/ + - uses: actions/upload-artifact@v3 - name: Upload carla-package artifact + name: Upload carla-package with: name: carla-package path: ${{ steps.extract_carla_package.outputs.destination }} + + # provide carla-python-api - uses: shrink/actions-docker-extract@v3 - name: extract PythonAPI package + name: Extract carla-python-api id: extract_python_api with: image: cgellerac/carla-source - output: /home/carla/PythonAPI + path: /home/carla/PythonAPI destination: /tmp/ + - uses: actions/upload-artifact@v3 - name: Upload carla-python-api artifact + name: Upload carla-python-api with: name: carla-python-api path: ${{ steps.extract_python_api.outputs.destination }} - - uses: actions/create-release@v1 + + # provide release + - uses: ncipollo/release-action@v1 name: Create Release - id: create_release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - draft: false - prerelease: false - - uses: actions/upload-release-asset@v1 - name: Upload Release Asset - id: upload-release-asset + if: startsWith(github.ref, 'refs/tags') with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ${{ steps.extract_python_api.outputs.destination }} - asset_name: carla-python-api + allowUpdates: true + tag: ${{ github.ref_name }} + commit: ${{ github.sha }} + artifacts: "${{ steps.extract_python_api.outputs.destination }}" + token: ${{ secrets.PAT }} + # create and push carla image create-carla-image: - name: create carla image + name: Create carla image runs-on: self-hosted - needs: create-carla-source-image + needs: provide-artifacts steps: + - uses: docker/login-action@v3 name: Login to Docker Hub with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Download artifact + + - name: Download carla-package uses: actions/download-artifact@v3 with: name: carla-package path: /tmp + - name: Extract artifact run: tar -xvzf /tmp/carla-package.tar.gz -C /build + - uses: docker/build-push-action@v5 name: Build and push with: - push: true file: /build/Dockerfile - context: /build tags: cgellerac/carla - \ No newline at end of file + context: /build + push: true \ No newline at end of file From 11a9ba6414e65e9fa84880cc56af108ac606bd06 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Wed, 13 Dec 2023 16:08:22 +0100 Subject: [PATCH 22/94] update extraction step --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 06742c73cab..074fddf51c6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -52,7 +52,7 @@ jobs: id: extract_carla_package with: image: cgellerac/carla-source - path: /home/carla/Dist/*.tar.gz # TODO: check if valid + path: /home/carla/Dist destination: /tmp/ - uses: actions/upload-artifact@v3 From f839c034fc6df2b89a85bb6a08935e7eca70d82b Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Wed, 13 Dec 2023 23:18:27 +0100 Subject: [PATCH 23/94] Update Carla.Dockerfile --- Util/Docker/Carla.Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Util/Docker/Carla.Dockerfile b/Util/Docker/Carla.Dockerfile index e474efbdec7..30fa27ba85f 100644 --- a/Util/Docker/Carla.Dockerfile +++ b/Util/Docker/Carla.Dockerfile @@ -15,7 +15,3 @@ RUN cd /home/carla/carla && \ make package WORKDIR /home/carla/carla - -COPY entrypoint.sh /usr/bin/ -ENTRYPOINT ["/usr/bin/entrypoint.sh"] -CMD [""] \ No newline at end of file From c2b6b50d4e49630cbdd13077a6f903205f7ca834 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Thu, 14 Dec 2023 09:28:26 +0100 Subject: [PATCH 24/94] add chown --- Util/Docker/Carla.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Util/Docker/Carla.Dockerfile b/Util/Docker/Carla.Dockerfile index 30fa27ba85f..7d39a33bd47 100644 --- a/Util/Docker/Carla.Dockerfile +++ b/Util/Docker/Carla.Dockerfile @@ -5,7 +5,7 @@ ARG GIT_BRANCH USER carla WORKDIR /home/carla -COPY . /home/carla/carla +COPY --chown=carla . /home/carla/carla RUN cd /home/carla/carla && \ ./Update.sh && \ From 8b49071da73687300d8651fbecc1afc475c3282d Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Thu, 14 Dec 2023 09:28:41 +0100 Subject: [PATCH 25/94] temporary disable prerequisites image --- .github/workflows/docker.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 074fddf51c6..cc3fa8d7b68 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,19 +5,19 @@ on: [push, pull_request] jobs: # create carla-prerequisites image - create-carla-prerequisites-image: - name: Create carla-prerequisites image - runs-on: self-hosted - steps: - - - uses: docker/build-push-action@v5 - name: Build image - with: - file: Util/Docker/Prerequisites.Dockerfile - tags: carla-prerequisites - build-args: | - EPIC_USER=${{ secrets.EPIC_USER }} - EPIC_PASS=${{ secrets.EPIC_PASS }} + #create-carla-prerequisites-image: + # name: Create carla-prerequisites image + # runs-on: self-hosted + # steps: + # + # - uses: docker/build-push-action@v5 + # name: Build image + # with: + # file: Util/Docker/Prerequisites.Dockerfile + # tags: carla-prerequisites + # build-args: | + # EPIC_USER=${{ secrets.EPIC_USER }} + # EPIC_PASS=${{ secrets.EPIC_PASS }} # create and push carla-source image create-carla-source-image: From 91898b74d6fd00a6528bd3853cf39d8bda1626f6 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Thu, 14 Dec 2023 09:29:38 +0100 Subject: [PATCH 26/94] update docker.yml --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cc3fa8d7b68..ee38c9aa867 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -23,7 +23,7 @@ jobs: create-carla-source-image: name: Create carla-source image runs-on: self-hosted - needs: create-carla-prerequisites-image + #needs: create-carla-prerequisites-image steps: - uses: docker/login-action@v3 From 6c107ddf3eb51bdd904da85622c36e63256073a0 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Thu, 14 Dec 2023 11:21:26 +0100 Subject: [PATCH 27/94] update Carla.Dockerfile --- Util/Docker/Carla.Dockerfile | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Util/Docker/Carla.Dockerfile b/Util/Docker/Carla.Dockerfile index 7d39a33bd47..55a9422e4fe 100644 --- a/Util/Docker/Carla.Dockerfile +++ b/Util/Docker/Carla.Dockerfile @@ -8,10 +8,18 @@ WORKDIR /home/carla COPY --chown=carla . /home/carla/carla RUN cd /home/carla/carla && \ - ./Update.sh && \ - make CarlaUE4Editor && \ - make PythonAPI ARGS="--python-version='3.10'" && \ - make build.utils && \ + ./Update.sh + +RUN cd /home/carla/carla && \ + make CarlaUE4Editor + +RUN cd /home/carla/carla && \ + make PythonAPI ARGS="--python-version='3.10'" + +RUN cd /home/carla/carla && \ + make build.utils + +RUN cd /home/carla/carla && \ make package WORKDIR /home/carla/carla From 66eaebe60d5788f2449df046dc48b9b819014189 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Thu, 14 Dec 2023 14:55:03 +0100 Subject: [PATCH 28/94] install python --- .github/workflows/docker.yml | 26 +++++++++++++------------- Util/Docker/Prerequisites.Dockerfile | 3 ++- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ee38c9aa867..d5b654c12aa 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,19 +5,19 @@ on: [push, pull_request] jobs: # create carla-prerequisites image - #create-carla-prerequisites-image: - # name: Create carla-prerequisites image - # runs-on: self-hosted - # steps: - # - # - uses: docker/build-push-action@v5 - # name: Build image - # with: - # file: Util/Docker/Prerequisites.Dockerfile - # tags: carla-prerequisites - # build-args: | - # EPIC_USER=${{ secrets.EPIC_USER }} - # EPIC_PASS=${{ secrets.EPIC_PASS }} + create-carla-prerequisites-image: + name: Create carla-prerequisites image + runs-on: self-hosted + steps: + + - uses: docker/build-push-action@v5 + name: Build image + with: + file: Util/Docker/Prerequisites.Dockerfile + tags: carla-prerequisites + build-args: | + EPIC_USER=${{ secrets.EPIC_USER }} + EPIC_PASS=${{ secrets.EPIC_PASS }} # create and push carla-source image create-carla-source-image: diff --git a/Util/Docker/Prerequisites.Dockerfile b/Util/Docker/Prerequisites.Dockerfile index 8d9e4da4b3b..e22245693e8 100644 --- a/Util/Docker/Prerequisites.Dockerfile +++ b/Util/Docker/Prerequisites.Dockerfile @@ -23,6 +23,7 @@ RUN apt-get update ; \ libvulkan-dev \ vulkan-tools \ libglvnd-dev \ + python \ python3.10 \ python3.10-dev \ python3.10-distutils \ @@ -54,7 +55,7 @@ RUN pip3.10 install -Iv setuptools==47.3.1 && \ # Set Alternatives and Defaults RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 && \ update-alternatives --set python3 /usr/bin/python3.10 && \ -update-alternatives --install /usr/bin/clang++ clang++ /usr/lib/llvm-8/bin/clang++ 180 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/lib/llvm-8/bin/clang++ 180 && \ update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-8/bin/clang 180 # Enable Vulkan support for NVIDIA GPUs From 6a6feb960177daabb0991fca35a873ed5e563981 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Thu, 14 Dec 2023 14:59:47 +0100 Subject: [PATCH 29/94] udpate docker.yml --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d5b654c12aa..bc5624cf34e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -23,7 +23,7 @@ jobs: create-carla-source-image: name: Create carla-source image runs-on: self-hosted - #needs: create-carla-prerequisites-image + needs: create-carla-prerequisites-image steps: - uses: docker/login-action@v3 From b777c6a37fbf2813d2774f7e7aa991ffd3302119 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Thu, 14 Dec 2023 18:26:47 +0100 Subject: [PATCH 30/94] add debugging output --- Util/BuildTools/Setup.sh | 6 ++++++ Util/Docker/Carla.Dockerfile | 15 +++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Util/BuildTools/Setup.sh b/Util/BuildTools/Setup.sh index 9aa65533bf0..ddc33dd36c3 100755 --- a/Util/BuildTools/Setup.sh +++ b/Util/BuildTools/Setup.sh @@ -711,9 +711,14 @@ else rm -Rf ${PATCHELF_TAR} rm -Rf ${PATCHELF_SOURCE_DIR} fi +log "Debugging Output 1" mkdir -p ${LIBCARLA_INSTALL_CLIENT_FOLDER}/bin/ cp ${PATCHELF_EXE} ${LIBCARLA_INSTALL_CLIENT_FOLDER}/bin/ +log "Debugging Output 2" + +log "${USE_PYTORCH}" +log "${USE_ROS2}" # ============================================================================== # -- Download libtorch and dependencies ---------------------------------------- @@ -874,6 +879,7 @@ fi # ============================================================================== # -- Generate Version.h -------------------------------------------------------- # ============================================================================== +log "Debugging Output 3" CARLA_VERSION=$(get_git_repository_version) diff --git a/Util/Docker/Carla.Dockerfile b/Util/Docker/Carla.Dockerfile index 55a9422e4fe..4ab6cecf1b4 100644 --- a/Util/Docker/Carla.Dockerfile +++ b/Util/Docker/Carla.Dockerfile @@ -11,15 +11,18 @@ RUN cd /home/carla/carla && \ ./Update.sh RUN cd /home/carla/carla && \ - make CarlaUE4Editor + make setup RUN cd /home/carla/carla && \ - make PythonAPI ARGS="--python-version='3.10'" + make CarlaUE4Editor + +#RUN cd /home/carla/carla && \ +# make PythonAPI ARGS="--python-version='3.10'" -RUN cd /home/carla/carla && \ - make build.utils +#RUN cd /home/carla/carla && \ +# make build.utils -RUN cd /home/carla/carla && \ - make package +#RUN cd /home/carla/carla && \ +# make package WORKDIR /home/carla/carla From 78a5c6a5b017419d4903c1a03ed349db13c7265d Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Thu, 14 Dec 2023 21:03:57 +0100 Subject: [PATCH 31/94] set path context to add .git folder --- .github/workflows/docker.yml | 39 ++++++++++++++++++++++-------------- Util/BuildTools/Setup.sh | 2 ++ Util/Docker/Carla.Dockerfile | 4 ++-- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index bc5624cf34e..6414968604a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,20 +4,20 @@ on: [push, pull_request] jobs: - # create carla-prerequisites image - create-carla-prerequisites-image: - name: Create carla-prerequisites image - runs-on: self-hosted - steps: - - - uses: docker/build-push-action@v5 - name: Build image - with: - file: Util/Docker/Prerequisites.Dockerfile - tags: carla-prerequisites - build-args: | - EPIC_USER=${{ secrets.EPIC_USER }} - EPIC_PASS=${{ secrets.EPIC_PASS }} + ## create carla-prerequisites image + #create-carla-prerequisites-image: + # name: Create carla-prerequisites image + # runs-on: self-hosted + # steps: + # + # - uses: docker/build-push-action@v5 + # name: Build image + # with: + # file: Util/Docker/Prerequisites.Dockerfile + # tags: carla-prerequisites + # build-args: | + # EPIC_USER=${{ secrets.EPIC_USER }} + # EPIC_PASS=${{ secrets.EPIC_PASS }} # create and push carla-source image create-carla-source-image: @@ -25,7 +25,15 @@ jobs: runs-on: self-hosted needs: create-carla-prerequisites-image steps: - + - uses: actions/checkout@v3 + name: Checkout + + - name: check folder + run: ls -la + + - uses: docker/setup-buildx-action@v3 + name: Set up Docker Buildx + - uses: docker/login-action@v3 name: Login to Docker Hub with: @@ -38,6 +46,7 @@ jobs: file: Util/Docker/Carla.Dockerfile tags: cgellerac/carla-source push: true + context: . # provide artifacts and releases provide-artifacts: diff --git a/Util/BuildTools/Setup.sh b/Util/BuildTools/Setup.sh index ddc33dd36c3..67f92588658 100755 --- a/Util/BuildTools/Setup.sh +++ b/Util/BuildTools/Setup.sh @@ -883,6 +883,8 @@ log "Debugging Output 3" CARLA_VERSION=$(get_git_repository_version) +log "Debugging Output 4" + log "CARLA version ${CARLA_VERSION}." VERSION_H_FILE=${LIBCARLA_ROOT_FOLDER}/source/carla/Version.h diff --git a/Util/Docker/Carla.Dockerfile b/Util/Docker/Carla.Dockerfile index 4ab6cecf1b4..c1829107cb1 100644 --- a/Util/Docker/Carla.Dockerfile +++ b/Util/Docker/Carla.Dockerfile @@ -13,8 +13,8 @@ RUN cd /home/carla/carla && \ RUN cd /home/carla/carla && \ make setup -RUN cd /home/carla/carla && \ - make CarlaUE4Editor +#RUN cd /home/carla/carla && \ +# make CarlaUE4Editor #RUN cd /home/carla/carla && \ # make PythonAPI ARGS="--python-version='3.10'" From eaf90f840daf3b6274c7b9efe3131bb7970b2d63 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Thu, 14 Dec 2023 21:07:46 +0100 Subject: [PATCH 32/94] fix --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6414968604a..f78cf707696 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -23,7 +23,7 @@ jobs: create-carla-source-image: name: Create carla-source image runs-on: self-hosted - needs: create-carla-prerequisites-image + #needs: create-carla-prerequisites-image steps: - uses: actions/checkout@v3 name: Checkout From 1340c1b50891d5dea3152fdaee4f49a74e427cfc Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Thu, 14 Dec 2023 21:33:07 +0100 Subject: [PATCH 33/94] add pre stage --- .github/workflows/docker.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f78cf707696..21ab1105c79 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,26 +4,26 @@ on: [push, pull_request] jobs: - ## create carla-prerequisites image - #create-carla-prerequisites-image: - # name: Create carla-prerequisites image - # runs-on: self-hosted - # steps: - # - # - uses: docker/build-push-action@v5 - # name: Build image - # with: - # file: Util/Docker/Prerequisites.Dockerfile - # tags: carla-prerequisites - # build-args: | - # EPIC_USER=${{ secrets.EPIC_USER }} - # EPIC_PASS=${{ secrets.EPIC_PASS }} + # create carla-prerequisites image + create-carla-prerequisites-image: + name: Create carla-prerequisites image + runs-on: self-hosted + steps: + + - uses: docker/build-push-action@v5 + name: Build image + with: + file: Util/Docker/Prerequisites.Dockerfile + tags: carla-prerequisites + build-args: | + EPIC_USER=${{ secrets.EPIC_USER }} + EPIC_PASS=${{ secrets.EPIC_PASS }} # create and push carla-source image create-carla-source-image: name: Create carla-source image runs-on: self-hosted - #needs: create-carla-prerequisites-image + needs: create-carla-prerequisites-image steps: - uses: actions/checkout@v3 name: Checkout From 432a643a4a94c668e2f8149b2f870e67e4696261 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Thu, 14 Dec 2023 22:34:59 +0100 Subject: [PATCH 34/94] remove buildx --- .github/workflows/docker.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 21ab1105c79..5b44362f189 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -31,9 +31,6 @@ jobs: - name: check folder run: ls -la - - uses: docker/setup-buildx-action@v3 - name: Set up Docker Buildx - - uses: docker/login-action@v3 name: Login to Docker Hub with: From 84bf749e88173474dd5d23dd103a741c1050bbe5 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Fri, 15 Dec 2023 08:44:09 +0100 Subject: [PATCH 35/94] clean up --- .github/workflows/docker.yml | 11 +++++------ Util/BuildTools/Setup.sh | 9 --------- Util/Docker/Carla.Dockerfile | 21 ++++++--------------- 3 files changed, 11 insertions(+), 30 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5b44362f189..8f34c7d976a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,9 +28,6 @@ jobs: - uses: actions/checkout@v3 name: Checkout - - name: check folder - run: ls -la - - uses: docker/login-action@v3 name: Login to Docker Hub with: @@ -55,10 +52,9 @@ jobs: # provide carla-package - uses: shrink/actions-docker-extract@v3 name: Extract carla-package - id: extract_carla_package with: image: cgellerac/carla-source - path: /home/carla/Dist + path: home/carla/carla/Dist destination: /tmp/ - uses: actions/upload-artifact@v3 @@ -73,7 +69,7 @@ jobs: id: extract_python_api with: image: cgellerac/carla-source - path: /home/carla/PythonAPI + path: home/carla/carla/PythonAPI destination: /tmp/ - uses: actions/upload-artifact@v3 @@ -112,6 +108,9 @@ jobs: name: carla-package path: /tmp + - name: debug + run: ls -la /tmp + - name: Extract artifact run: tar -xvzf /tmp/carla-package.tar.gz -C /build diff --git a/Util/BuildTools/Setup.sh b/Util/BuildTools/Setup.sh index 67f92588658..d3702e4ddf2 100755 --- a/Util/BuildTools/Setup.sh +++ b/Util/BuildTools/Setup.sh @@ -711,14 +711,9 @@ else rm -Rf ${PATCHELF_TAR} rm -Rf ${PATCHELF_SOURCE_DIR} fi -log "Debugging Output 1" mkdir -p ${LIBCARLA_INSTALL_CLIENT_FOLDER}/bin/ cp ${PATCHELF_EXE} ${LIBCARLA_INSTALL_CLIENT_FOLDER}/bin/ -log "Debugging Output 2" - -log "${USE_PYTORCH}" -log "${USE_ROS2}" # ============================================================================== # -- Download libtorch and dependencies ---------------------------------------- @@ -879,12 +874,8 @@ fi # ============================================================================== # -- Generate Version.h -------------------------------------------------------- # ============================================================================== -log "Debugging Output 3" - CARLA_VERSION=$(get_git_repository_version) -log "Debugging Output 4" - log "CARLA version ${CARLA_VERSION}." VERSION_H_FILE=${LIBCARLA_ROOT_FOLDER}/source/carla/Version.h diff --git a/Util/Docker/Carla.Dockerfile b/Util/Docker/Carla.Dockerfile index c1829107cb1..bf9733e6b4a 100644 --- a/Util/Docker/Carla.Dockerfile +++ b/Util/Docker/Carla.Dockerfile @@ -3,26 +3,17 @@ FROM carla-prerequisites:latest ARG GIT_BRANCH USER carla -WORKDIR /home/carla COPY --chown=carla . /home/carla/carla -RUN cd /home/carla/carla && \ - ./Update.sh +WORKDIR /home/carla/carla -RUN cd /home/carla/carla && \ - make setup +RUN ./Update.sh -#RUN cd /home/carla/carla && \ -# make CarlaUE4Editor +RUN make CarlaUE4Editor -#RUN cd /home/carla/carla && \ -# make PythonAPI ARGS="--python-version='3.10'" - -#RUN cd /home/carla/carla && \ -# make build.utils +RUN make PythonAPI ARGS="--python-version='3.10'" -#RUN cd /home/carla/carla && \ -# make package +RUN make build.utils -WORKDIR /home/carla/carla +RUN make package \ No newline at end of file From 9d2efe59e71aecd3d715966ec8dbded0edfc1f9b Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Fri, 15 Dec 2023 12:43:10 +0100 Subject: [PATCH 36/94] install setuptools --- Util/Docker/Carla.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Util/Docker/Carla.Dockerfile b/Util/Docker/Carla.Dockerfile index bf9733e6b4a..0dca6f936e6 100644 --- a/Util/Docker/Carla.Dockerfile +++ b/Util/Docker/Carla.Dockerfile @@ -8,6 +8,8 @@ COPY --chown=carla . /home/carla/carla WORKDIR /home/carla/carla +RUN pip3 install -Iv setuptools==47.3.1 + RUN ./Update.sh RUN make CarlaUE4Editor From 938f4a8ed28aea56ee2f411fdbd169de0dc38d2f Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Sat, 16 Dec 2023 10:59:23 +0100 Subject: [PATCH 37/94] Update BuildUtilsDocker.sh --- Util/BuildTools/BuildUtilsDocker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Util/BuildTools/BuildUtilsDocker.sh b/Util/BuildTools/BuildUtilsDocker.sh index 609d2b5ff42..c1aa6f3de8c 100755 --- a/Util/BuildTools/BuildUtilsDocker.sh +++ b/Util/BuildTools/BuildUtilsDocker.sh @@ -22,7 +22,7 @@ if [ ! -d "${FBX2OBJ_DEP_FOLDER}" ]; then echo "Unpacking..." mkdir -p "${FBX2OBJ_DEP_FOLDER}" unzip "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}.zip" -d "${FBX2OBJ_DEP_FOLDER}" - rm "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}.tar.gz" + rm "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}.zip" fi From e824d76fbba192a2f34c96893de5995ba70086c6 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Sun, 17 Dec 2023 11:28:15 +0100 Subject: [PATCH 38/94] add debugging --- .github/workflows/docker.yml | 78 +++++++++++++++--------------------- 1 file changed, 33 insertions(+), 45 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8f34c7d976a..88aef565099 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,49 +4,11 @@ on: [push, pull_request] jobs: - # create carla-prerequisites image - create-carla-prerequisites-image: - name: Create carla-prerequisites image - runs-on: self-hosted - steps: - - - uses: docker/build-push-action@v5 - name: Build image - with: - file: Util/Docker/Prerequisites.Dockerfile - tags: carla-prerequisites - build-args: | - EPIC_USER=${{ secrets.EPIC_USER }} - EPIC_PASS=${{ secrets.EPIC_PASS }} - - # create and push carla-source image - create-carla-source-image: - name: Create carla-source image - runs-on: self-hosted - needs: create-carla-prerequisites-image - steps: - - uses: actions/checkout@v3 - name: Checkout - - - uses: docker/login-action@v3 - name: Login to Docker Hub - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - uses: docker/build-push-action@v5 - name: Build and push image - with: - file: Util/Docker/Carla.Dockerfile - tags: cgellerac/carla-source - push: true - context: . - # provide artifacts and releases provide-artifacts: name: Provide carla artifacts runs-on: self-hosted - needs: create-carla-source-image + #needs: create-carla-source-image steps: # provide carla-package @@ -57,12 +19,18 @@ jobs: path: home/carla/carla/Dist destination: /tmp/ - - uses: actions/upload-artifact@v3 - name: Upload carla-package - with: - name: carla-package - path: ${{ steps.extract_carla_package.outputs.destination }} + - name: test1 + run: pwd + - name: test2 + run: ls + + - name: test3 + run: ls /tmp + + - name: test4 + run: echo ${{ steps.extract_carla_package.outputs.destination }} + # provide carla-python-api - uses: shrink/actions-docker-extract@v3 name: Extract carla-python-api @@ -72,6 +40,26 @@ jobs: path: home/carla/carla/PythonAPI destination: /tmp/ + - name: test5 + run: pwd + + - name: test6 + run: ls + + - name: test7 + run: ls /tmp + + - name: test8 + run: echo ${{ steps.extract_python_api.outputs.destination }} + + + + - uses: actions/upload-artifact@v3 + name: Upload carla-package + with: + name: carla-package + path: ${{ steps.extract_carla_package.outputs.destination }} + - uses: actions/upload-artifact@v3 name: Upload carla-python-api with: @@ -120,4 +108,4 @@ jobs: file: /build/Dockerfile tags: cgellerac/carla context: /build - push: true \ No newline at end of file + push: true From 638082be9445041ba248fb20dd912d8d046edc15 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Sun, 17 Dec 2023 11:38:08 +0100 Subject: [PATCH 39/94] Update docker.yml --- .github/workflows/docker.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 88aef565099..d248d565da4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,7 +17,7 @@ jobs: with: image: cgellerac/carla-source path: home/carla/carla/Dist - destination: /tmp/ + destination: artifacts/ - name: test1 run: pwd @@ -38,7 +38,7 @@ jobs: with: image: cgellerac/carla-source path: home/carla/carla/PythonAPI - destination: /tmp/ + destination: artifacts/ - name: test5 run: pwd @@ -58,13 +58,13 @@ jobs: name: Upload carla-package with: name: carla-package - path: ${{ steps.extract_carla_package.outputs.destination }} + path: artifacts/Dist - uses: actions/upload-artifact@v3 name: Upload carla-python-api with: name: carla-python-api - path: ${{ steps.extract_python_api.outputs.destination }} + path: artifacts/PythonAPI # provide release - uses: ncipollo/release-action@v1 @@ -74,7 +74,7 @@ jobs: allowUpdates: true tag: ${{ github.ref_name }} commit: ${{ github.sha }} - artifacts: "${{ steps.extract_python_api.outputs.destination }}" + artifacts: artifacts/PythonAPI token: ${{ secrets.PAT }} # create and push carla image From a6ea5ece3978f7422a8440b0a08807a31c0e8570 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Sun, 17 Dec 2023 11:47:47 +0100 Subject: [PATCH 40/94] Update docker.yml --- .github/workflows/docker.yml | 46 ++++++++---------------------------- 1 file changed, 10 insertions(+), 36 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d248d565da4..e8826d17b63 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -18,19 +18,13 @@ jobs: image: cgellerac/carla-source path: home/carla/carla/Dist destination: artifacts/ - - - name: test1 - run: pwd - - - name: test2 - run: ls - - - name: test3 - run: ls /tmp - - - name: test4 - run: echo ${{ steps.extract_carla_package.outputs.destination }} - + + - uses: actions/upload-artifact@v3 + name: Upload carla-package + with: + name: carla-package + path: artifacts/Dist + # provide carla-python-api - uses: shrink/actions-docker-extract@v3 name: Extract carla-python-api @@ -40,26 +34,6 @@ jobs: path: home/carla/carla/PythonAPI destination: artifacts/ - - name: test5 - run: pwd - - - name: test6 - run: ls - - - name: test7 - run: ls /tmp - - - name: test8 - run: echo ${{ steps.extract_python_api.outputs.destination }} - - - - - uses: actions/upload-artifact@v3 - name: Upload carla-package - with: - name: carla-package - path: artifacts/Dist - - uses: actions/upload-artifact@v3 name: Upload carla-python-api with: @@ -94,13 +68,13 @@ jobs: uses: actions/download-artifact@v3 with: name: carla-package - path: /tmp + path: artifacts - name: debug - run: ls -la /tmp + run: ls -la artifacts - name: Extract artifact - run: tar -xvzf /tmp/carla-package.tar.gz -C /build + run: tar -xvzf artifacts/*.tar.gz -C /build - uses: docker/build-push-action@v5 name: Build and push From 4c43e2fa3a9afb24e23b8cd4491a531d302e975e Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Sun, 17 Dec 2023 19:40:06 +0100 Subject: [PATCH 41/94] temporary delete provide job --- .github/workflows/docker.yml | 65 +++++------------------------------- 1 file changed, 9 insertions(+), 56 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e8826d17b63..a1a6e8be65f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,58 +4,11 @@ on: [push, pull_request] jobs: - # provide artifacts and releases - provide-artifacts: - name: Provide carla artifacts - runs-on: self-hosted - #needs: create-carla-source-image - steps: - - # provide carla-package - - uses: shrink/actions-docker-extract@v3 - name: Extract carla-package - with: - image: cgellerac/carla-source - path: home/carla/carla/Dist - destination: artifacts/ - - - uses: actions/upload-artifact@v3 - name: Upload carla-package - with: - name: carla-package - path: artifacts/Dist - - # provide carla-python-api - - uses: shrink/actions-docker-extract@v3 - name: Extract carla-python-api - id: extract_python_api - with: - image: cgellerac/carla-source - path: home/carla/carla/PythonAPI - destination: artifacts/ - - - uses: actions/upload-artifact@v3 - name: Upload carla-python-api - with: - name: carla-python-api - path: artifacts/PythonAPI - - # provide release - - uses: ncipollo/release-action@v1 - name: Create Release - if: startsWith(github.ref, 'refs/tags') - with: - allowUpdates: true - tag: ${{ github.ref_name }} - commit: ${{ github.sha }} - artifacts: artifacts/PythonAPI - token: ${{ secrets.PAT }} - # create and push carla image create-carla-image: name: Create carla image runs-on: self-hosted - needs: provide-artifacts + #needs: provide-artifacts steps: - uses: docker/login-action@v3 @@ -64,22 +17,22 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Download carla-package - uses: actions/download-artifact@v3 - with: - name: carla-package - path: artifacts + #- name: Download carla-package + # uses: actions/download-artifact@v3 + # with: + # name: carla-package + # path: artifacts - name: debug run: ls -la artifacts - name: Extract artifact - run: tar -xvzf artifacts/*.tar.gz -C /build + run: tar -xvzf artifacts/*.tar.gz -C build - uses: docker/build-push-action@v5 name: Build and push with: - file: /build/Dockerfile + file: build/Dockerfile tags: cgellerac/carla - context: /build + context: build push: true From e960c2bbaa789b422d4c8b0ac2850f1f4c201b53 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Sun, 17 Dec 2023 19:42:44 +0100 Subject: [PATCH 42/94] Update docker.yml --- .github/workflows/docker.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a1a6e8be65f..00d70fe32b8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,10 +25,16 @@ jobs: - name: debug run: ls -la artifacts + + - name: debug2 + run: mkdir build - name: Extract artifact run: tar -xvzf artifacts/*.tar.gz -C build + - name: debug3 + run: ls -la build + - uses: docker/build-push-action@v5 name: Build and push with: From 5847c432d948a2a0d013beb1e80d59a336e05bde Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Sun, 17 Dec 2023 21:43:51 +0100 Subject: [PATCH 43/94] Update docker.yml --- .github/workflows/docker.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 00d70fe32b8..b3a9a16aed7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,12 +25,9 @@ jobs: - name: debug run: ls -la artifacts - - - name: debug2 - run: mkdir build - name: Extract artifact - run: tar -xvzf artifacts/*.tar.gz -C build + run: mkdir build && tar -xvzf artifacts/*.tar.gz -C build - name: debug3 run: ls -la build From 3548be662deda09430bd68952772b0c807de6769 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Sun, 17 Dec 2023 21:47:54 +0100 Subject: [PATCH 44/94] Update docker.yml --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b3a9a16aed7..c1630a7ad73 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,7 +27,7 @@ jobs: run: ls -la artifacts - name: Extract artifact - run: mkdir build && tar -xvzf artifacts/*.tar.gz -C build + run: rm -rf build && mkdir build && tar -xvzf artifacts/*.tar.gz -C build - name: debug3 run: ls -la build From 2c3d2af5489ff9cc82c43c592cd77e92e360b1b6 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Sun, 17 Dec 2023 22:00:44 +0100 Subject: [PATCH 45/94] Restructure final docker.yml --- .github/workflows/docker.yml | 106 ++++++++++++++++++++++++++++++++--- 1 file changed, 97 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c1630a7ad73..2b986d1e2e1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,13 +4,107 @@ on: [push, pull_request] jobs: + clean-up: + name: Clean up + runs-on: self-hosted + steps: + + - name: Clean up + run: rm -rf * + + # create carla-prerequisites image + create-carla-prerequisites-image: + name: Create carla-prerequisites image + runs-on: self-hosted + needs: clean-up + steps: + + - uses: docker/build-push-action@v5 + name: Build image + with: + file: Util/Docker/Prerequisites.Dockerfile + tags: carla-prerequisites + build-args: | + EPIC_USER=${{ secrets.EPIC_USER }} + EPIC_PASS=${{ secrets.EPIC_PASS }} + + # create and push carla-source image + create-carla-source-image: + name: Create carla-source image + runs-on: self-hosted + needs: create-carla-prerequisites-image + steps: + - uses: actions/checkout@v3 + name: Checkout + + - uses: docker/login-action@v3 + name: Login to Docker Hub + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - uses: docker/build-push-action@v5 + name: Build and push image + with: + file: Util/Docker/Carla.Dockerfile + tags: cgellerac/carla-source + push: true + context: . + + # provide artifacts and releases + provide-artifacts: + name: Provide carla artifacts + runs-on: self-hosted + needs: create-carla-source-image + steps: + + # provide carla-package + - uses: shrink/actions-docker-extract@v3 + name: Extract carla-package + with: + image: cgellerac/carla-source + path: home/carla/carla/Dist + destination: artifacts/ + + #- uses: actions/upload-artifact@v3 + # name: Upload carla-package + # with: + # name: carla-package + # path: artifacts/Dist + + # provide carla-python-api + - uses: shrink/actions-docker-extract@v3 + name: Extract carla-python-api + id: extract_python_api + with: + image: cgellerac/carla-source + path: home/carla/carla/PythonAPI + destination: artifacts/ + + #- uses: actions/upload-artifact@v3 + # name: Upload carla-python-api + # with: + # name: carla-python-api + # path: artifacts/PythonAPI + + # provide release + - uses: ncipollo/release-action@v1 + name: Create Release + if: startsWith(github.ref, 'refs/tags') + with: + allowUpdates: true + tag: ${{ github.ref_name }} + commit: ${{ github.sha }} + artifacts: artifacts/PythonAPI + token: ${{ secrets.PAT }} + # create and push carla image create-carla-image: name: Create carla image runs-on: self-hosted - #needs: provide-artifacts + needs: provide-artifacts steps: - + - uses: docker/login-action@v3 name: Login to Docker Hub with: @@ -23,14 +117,8 @@ jobs: # name: carla-package # path: artifacts - - name: debug - run: ls -la artifacts - - name: Extract artifact - run: rm -rf build && mkdir build && tar -xvzf artifacts/*.tar.gz -C build - - - name: debug3 - run: ls -la build + run: tar -xvzf artifacts/*.tar.gz -C build - uses: docker/build-push-action@v5 name: Build and push From e29c286702d07e51cbac458ec3204087e56733c9 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Mon, 18 Dec 2023 09:21:20 +0100 Subject: [PATCH 46/94] Fixed pipeline --- .github/workflows/docker.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2b986d1e2e1..92c2658b46e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -66,6 +66,9 @@ jobs: path: home/carla/carla/Dist destination: artifacts/ + - name: Keep only .tar.gz archive + run: rm -rf artifacts/Dist/CARLA_Shipping_* + #- uses: actions/upload-artifact@v3 # name: Upload carla-package # with: @@ -118,7 +121,7 @@ jobs: # path: artifacts - name: Extract artifact - run: tar -xvzf artifacts/*.tar.gz -C build + run: tar -xvzf artifacts/Dist/*.tar.gz -C build - uses: docker/build-push-action@v5 name: Build and push From fd31f867825e01f59c09eefca434015e7fed09c9 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Mon, 18 Dec 2023 10:49:10 +0100 Subject: [PATCH 47/94] clean up shipping folder --- .github/workflows/docker.yml | 3 --- Util/Docker/Carla.Dockerfile | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 92c2658b46e..71f01c0337e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -66,9 +66,6 @@ jobs: path: home/carla/carla/Dist destination: artifacts/ - - name: Keep only .tar.gz archive - run: rm -rf artifacts/Dist/CARLA_Shipping_* - #- uses: actions/upload-artifact@v3 # name: Upload carla-package # with: diff --git a/Util/Docker/Carla.Dockerfile b/Util/Docker/Carla.Dockerfile index 0dca6f936e6..616333cf4b4 100644 --- a/Util/Docker/Carla.Dockerfile +++ b/Util/Docker/Carla.Dockerfile @@ -18,4 +18,4 @@ RUN make PythonAPI ARGS="--python-version='3.10'" RUN make build.utils -RUN make package \ No newline at end of file +RUN make package ARGS="--clean-intermediate" \ No newline at end of file From 2550acb20d13f60036658ea56fba408661fc2f1a Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Mon, 18 Dec 2023 10:55:17 +0100 Subject: [PATCH 48/94] beautify --- Util/BuildTools/Setup.sh | 1 + Util/Docker/Prerequisites.Dockerfile | 3 --- Util/Docker/Release.Dockerfile | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Util/BuildTools/Setup.sh b/Util/BuildTools/Setup.sh index d3702e4ddf2..9aa65533bf0 100755 --- a/Util/BuildTools/Setup.sh +++ b/Util/BuildTools/Setup.sh @@ -874,6 +874,7 @@ fi # ============================================================================== # -- Generate Version.h -------------------------------------------------------- # ============================================================================== + CARLA_VERSION=$(get_git_repository_version) log "CARLA version ${CARLA_VERSION}." diff --git a/Util/Docker/Prerequisites.Dockerfile b/Util/Docker/Prerequisites.Dockerfile index e22245693e8..0c11b6b05a1 100644 --- a/Util/Docker/Prerequisites.Dockerfile +++ b/Util/Docker/Prerequisites.Dockerfile @@ -1,13 +1,10 @@ FROM nvidia/cuda:11.3.1-runtime-ubuntu20.04 -# TODO: Change to 22.04 USER root ARG EPIC_USER=user ARG EPIC_PASS=pass ENV DEBIAN_FRONTEND=noninteractive - -# Install dependencies RUN apt-get update ; \ apt-get install -y wget software-properties-common && \ add-apt-repository ppa:ubuntu-toolchain-r/test && \ diff --git a/Util/Docker/Release.Dockerfile b/Util/Docker/Release.Dockerfile index 3cb65be0469..c0737b6e5cd 100644 --- a/Util/Docker/Release.Dockerfile +++ b/Util/Docker/Release.Dockerfile @@ -4,7 +4,6 @@ # sudo -E docker run --rm --gpus all -it --net=host carla:latest /bin/bash FROM nvidia/vulkan:1.1.121-cuda-10.1--ubuntu18.04 -# TODO: Change to 22.04 RUN apt-key adv --fetch-keys "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub" From 69efa8b356f4cc55bff7a3018dcd4b95c8da859e Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Mon, 18 Dec 2023 10:55:57 +0100 Subject: [PATCH 49/94] beautify --- Util/Docker/Release.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Util/Docker/Release.Dockerfile b/Util/Docker/Release.Dockerfile index c0737b6e5cd..39d4b6d47a9 100644 --- a/Util/Docker/Release.Dockerfile +++ b/Util/Docker/Release.Dockerfile @@ -3,7 +3,7 @@ # # sudo -E docker run --rm --gpus all -it --net=host carla:latest /bin/bash -FROM nvidia/vulkan:1.1.121-cuda-10.1--ubuntu18.04 +FROM nvidia/vulkan:1.1.121-cuda-10.1--ubuntu18.04 RUN apt-key adv --fetch-keys "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub" From 2b69811806ef4feb9a76daa3855d3c4223bbb6f4 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Mon, 18 Dec 2023 22:07:27 +0100 Subject: [PATCH 50/94] Update Carla.Dockerfile --- Util/Docker/Carla.Dockerfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Util/Docker/Carla.Dockerfile b/Util/Docker/Carla.Dockerfile index 616333cf4b4..a9343bed201 100644 --- a/Util/Docker/Carla.Dockerfile +++ b/Util/Docker/Carla.Dockerfile @@ -11,11 +11,7 @@ WORKDIR /home/carla/carla RUN pip3 install -Iv setuptools==47.3.1 RUN ./Update.sh - RUN make CarlaUE4Editor - RUN make PythonAPI ARGS="--python-version='3.10'" - RUN make build.utils - -RUN make package ARGS="--clean-intermediate" \ No newline at end of file +RUN make package ARGS="--clean-intermediate" From 663f7959e98e1b32946a7032b2f7f7a0a208bbc0 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Tue, 19 Dec 2023 07:49:16 +0100 Subject: [PATCH 51/94] Update docker.yml --- .github/workflows/docker.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 71f01c0337e..dcfe7a1bde5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,6 +24,7 @@ jobs: with: file: Util/Docker/Prerequisites.Dockerfile tags: carla-prerequisites + no-cache: true build-args: | EPIC_USER=${{ secrets.EPIC_USER }} EPIC_PASS=${{ secrets.EPIC_PASS }} @@ -48,6 +49,7 @@ jobs: with: file: Util/Docker/Carla.Dockerfile tags: cgellerac/carla-source + no-cache: true push: true context: . @@ -125,5 +127,6 @@ jobs: with: file: build/Dockerfile tags: cgellerac/carla + no-cache: true context: build push: true From 8b95c2893fa78b36aeb5a8575522a5e94741662d Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Tue, 19 Dec 2023 16:14:04 +0100 Subject: [PATCH 52/94] Add mkdir build --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index dcfe7a1bde5..2b717d720c8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -120,7 +120,7 @@ jobs: # path: artifacts - name: Extract artifact - run: tar -xvzf artifacts/Dist/*.tar.gz -C build + run: mkdir build && tar -xvzf artifacts/Dist/*.tar.gz -C build - uses: docker/build-push-action@v5 name: Build and push From 9cec4dbc024bdd6f3e4201ff06a5824e53e919e7 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Wed, 20 Dec 2023 09:02:12 +0100 Subject: [PATCH 53/94] Create archive --- .github/workflows/docker.yml | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2b717d720c8..c6d619fa169 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -68,12 +68,6 @@ jobs: path: home/carla/carla/Dist destination: artifacts/ - #- uses: actions/upload-artifact@v3 - # name: Upload carla-package - # with: - # name: carla-package - # path: artifacts/Dist - # provide carla-python-api - uses: shrink/actions-docker-extract@v3 name: Extract carla-python-api @@ -82,13 +76,12 @@ jobs: image: cgellerac/carla-source path: home/carla/carla/PythonAPI destination: artifacts/ - - #- uses: actions/upload-artifact@v3 - # name: Upload carla-python-api - # with: - # name: carla-python-api - # path: artifacts/PythonAPI - + + # create archive for release + - name: Create archive + if: startsWith(github.ref, 'refs/tags') + run: tar -czvf artifacts/PythonAPI.tar.gz artifacts/PythonAPI + # provide release - uses: ncipollo/release-action@v1 name: Create Release @@ -97,7 +90,7 @@ jobs: allowUpdates: true tag: ${{ github.ref_name }} commit: ${{ github.sha }} - artifacts: artifacts/PythonAPI + artifacts: artifacts/PythonAPI.tar.gz token: ${{ secrets.PAT }} # create and push carla image @@ -113,12 +106,6 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - #- name: Download carla-package - # uses: actions/download-artifact@v3 - # with: - # name: carla-package - # path: artifacts - - name: Extract artifact run: mkdir build && tar -xvzf artifacts/Dist/*.tar.gz -C build From e5a9dc409ae6c1769a5aa7a6630c690f5b9f5e62 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Wed, 20 Dec 2023 09:09:25 +0100 Subject: [PATCH 54/94] Update docker.yml --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c6d619fa169..30f3b6016ac 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -107,7 +107,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Extract artifact - run: mkdir build && tar -xvzf artifacts/Dist/*.tar.gz -C build + run: mkdir -p build && tar -xvzf artifacts/Dist/*.tar.gz -C build - uses: docker/build-push-action@v5 name: Build and push From 6fda33c0ff3e1a25bc4505c625a381b54e2bd925 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Thu, 21 Dec 2023 14:02:19 +0100 Subject: [PATCH 55/94] Update docker.yml --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 30f3b6016ac..ad825389d14 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,6 +4,7 @@ on: [push, pull_request] jobs: + # clean up clean-up: name: Clean up runs-on: self-hosted From 79a6c629e3f60b772d3d2a4da8b7d0438d0076da Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Thu, 21 Dec 2023 14:05:23 +0100 Subject: [PATCH 56/94] Update docker.yml --- .github/workflows/docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ad825389d14..03e43895a75 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -38,7 +38,9 @@ jobs: steps: - uses: actions/checkout@v3 name: Checkout - + with: + submodules: true + - uses: docker/login-action@v3 name: Login to Docker Hub with: From d678ac782a775d4a5de121109c2a900ac1adedb2 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Thu, 21 Dec 2023 14:06:28 +0100 Subject: [PATCH 57/94] Update docker.yml --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 03e43895a75..55365b80b87 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -37,7 +37,7 @@ jobs: needs: create-carla-prerequisites-image steps: - uses: actions/checkout@v3 - name: Checkout + name: Checkout repository with: submodules: true From 48e593a2f15050048ca7b517f763d0f7fdf12292 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Thu, 21 Dec 2023 15:25:39 +0100 Subject: [PATCH 58/94] update readme --- README.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 98086e0c0c3..ec1664c3e33 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,18 @@ -CARLA Simulator -=============== +# Simulation Core: *carla-simulator* + +

+ + + + + +

+ +We recommend to use the *carla-simulator* as **simulation core** within our proposed simulation framework . + +--- +--- +## Original README [![Build Status](https://travis-ci.org/carla-simulator/carla.svg?branch=master)](https://travis-ci.org/carla-simulator/carla) [![Documentation](https://readthedocs.org/projects/carla/badge/?version=latest)](http://carla.readthedocs.io) From a6f5af0574043b3dd2b72c3bb9a0f13106e0d33e Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Thu, 21 Dec 2023 15:38:34 +0100 Subject: [PATCH 59/94] update readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ec1664c3e33..a596c830622 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@

-We recommend to use the *carla-simulator* as **simulation core** within our proposed simulation framework . +We recommend to use the *carla-simulator* as **simulation core** within our proposed simulation framework carla-compose. + --- --- From 997b38e10a0d749ac32074580e63a328cccc09de Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Thu, 21 Dec 2023 15:59:40 +0100 Subject: [PATCH 60/94] Rename images --- .github/workflows/docker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 55365b80b87..f4688df252d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -51,7 +51,7 @@ jobs: name: Build and push image with: file: Util/Docker/Carla.Dockerfile - tags: cgellerac/carla-source + tags: cgellerac/carla-simulator:source no-cache: true push: true context: . @@ -67,7 +67,7 @@ jobs: - uses: shrink/actions-docker-extract@v3 name: Extract carla-package with: - image: cgellerac/carla-source + image: cgellerac/carla-simulator:source path: home/carla/carla/Dist destination: artifacts/ @@ -76,7 +76,7 @@ jobs: name: Extract carla-python-api id: extract_python_api with: - image: cgellerac/carla-source + image: cgellerac/carla-simulator:source path: home/carla/carla/PythonAPI destination: artifacts/ @@ -116,7 +116,7 @@ jobs: name: Build and push with: file: build/Dockerfile - tags: cgellerac/carla + tags: cgellerac/carla-simulator no-cache: true context: build push: true From 0614639850729d1287fc3c62f05be702317c030c Mon Sep 17 00:00:00 2001 From: Benedikt Haas Date: Fri, 5 Jan 2024 15:05:27 +0100 Subject: [PATCH 61/94] create ping script for server healthchecks --- PythonAPI/util/ping.py | 69 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 PythonAPI/util/ping.py diff --git a/PythonAPI/util/ping.py b/PythonAPI/util/ping.py new file mode 100644 index 00000000000..158d1877b9c --- /dev/null +++ b/PythonAPI/util/ping.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +# Copyright (c) 2019 Computer Vision Center (CVC) at the Universitat Autonoma de +# Barcelona (UAB). +# +# This work is licensed under the terms of the MIT license. +# For a copy, see . + +"""Performs a single connection check to the simulator.""" + +import glob +import os +import sys + +try: + sys.path.append(glob.glob('../carla/dist/carla-*%d.%d-%s.egg' % ( + sys.version_info.major, + sys.version_info.minor, + 'win-amd64' if os.name == 'nt' else 'linux-x86_64'))[0]) +except IndexError: + pass + + +import carla + +import argparse + + +def main(): + argparser = argparse.ArgumentParser( + description=__doc__) + argparser.add_argument( + '--host', + metavar='H', + default='127.0.0.1', + help='IP of the host server (default: 127.0.0.1)') + argparser.add_argument( + '-p', '--port', + metavar='P', + default=2000, + type=int, + help='TCP port to listen to (default: 2000)') + argparser.add_argument( + '--timeout', + metavar='T', + default=1.0, + type=float, + help='time-out in seconds (default: 1)') + args = argparser.parse_args() + + try: + client = carla.Client(args.host, args.port) + except RuntimeError: + print('IP/Hostname %s could not be resolved' % (args.host)) + return 1 + + client.set_timeout(args.timeout) + + try: + print('CARLA %s connected at %s:%d.' % (client.get_server_version(), args.host, args.port)) + return 0 + except RuntimeError: + print('Failed to connect to %s:%d before time-out of %d second(s).' % (args.host, args.port, args.timeout)) + return 1 + + +if __name__ == '__main__': + + sys.exit(main()) From c209e219821b791e0b87f901345c84810b1f93c3 Mon Sep 17 00:00:00 2001 From: Benedikt Haas Date: Fri, 5 Jan 2024 15:49:16 +0100 Subject: [PATCH 62/94] create updated vulkan base image dockerfile --- Util/Docker/vulkan-base/Vulkan.Dockerfile | 49 +++++++++++++++++++++++ Util/Docker/vulkan-base/nvidia_icd.json | 7 ++++ 2 files changed, 56 insertions(+) create mode 100644 Util/Docker/vulkan-base/Vulkan.Dockerfile create mode 100644 Util/Docker/vulkan-base/nvidia_icd.json diff --git a/Util/Docker/vulkan-base/Vulkan.Dockerfile b/Util/Docker/vulkan-base/Vulkan.Dockerfile new file mode 100644 index 00000000000..793e1b79688 --- /dev/null +++ b/Util/Docker/vulkan-base/Vulkan.Dockerfile @@ -0,0 +1,49 @@ +# custom Dockerfile for the release base image (based off of discontinued https://gitlab.com/nvidia/container-images/vulkan ) + +ARG DEBIAN_FRONTEND=noninteractive +ARG BASE_DIST=ubuntu22.04 +ARG CUDA_VERSION=12.2.0 + +FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libglvnd0 \ + libgl1 \ + libglx0 \ + libegl1 \ + libgles2 \ + libxcb1-dev \ + libjpeg8 \ + libtiff5 \ + wget \ + xz-utils \ + && rm -rf /var/lib/apt/lists/* + +# Install Vulkan SDK +ARG VULKAN_SDK_VERSION=1.3.268.0 +#set VULKAN_SDK_VERSION as latest via build-arg=`curl https://vulkan.lunarg.com/sdk/latest/linux.txt` +ARG VULKAN_SDK_VERSION +# Download the Vulkan SDK and extract the headers, loaders, layers and binary utilities +RUN wget -q --show-progress \ + --progress=bar:force:noscroll \ + https://sdk.lunarg.com/sdk/download/latest/linux/vulkan_sdk.tar.gz \ + -O /tmp/vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.gz \ + && echo "Installing Vulkan SDK ${VULKAN_SDK_VERSION}" \ + && mkdir -p /opt/vulkan \ + && tar -xf /tmp/vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.gz -C /opt/vulkan \ + && mkdir -p /usr/local/include/ && cp -ra /opt/vulkan/${VULKAN_SDK_VERSION}/x86_64/include/* /usr/local/include/ \ + && mkdir -p /usr/local/lib && cp -ra /opt/vulkan/${VULKAN_SDK_VERSION}/x86_64/lib/* /usr/local/lib/ \ + && cp -a /opt/vulkan/${VULKAN_SDK_VERSION}/x86_64/lib/libVkLayer_*.so /usr/local/lib \ + && mkdir -p /usr/local/share/vulkan/explicit_layer.d \ + && cp /opt/vulkan/${VULKAN_SDK_VERSION}/x86_64/etc/vulkan/explicit_layer.d/VkLayer_*.json /usr/local/share/vulkan/explicit_layer.d \ + && mkdir -p /usr/local/share/vulkan/registry \ + && cp -a /opt/vulkan/${VULKAN_SDK_VERSION}/x86_64/share/vulkan/registry/* /usr/local/share/vulkan/registry \ + && cp -a /opt/vulkan/${VULKAN_SDK_VERSION}/x86_64/bin/* /usr/local/bin \ + && ldconfig \ + && rm /tmp/vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.gz && rm -rf /opt/vulkan + +# Copy in Nvidia driver config +COPY nvidia_icd.json /etc/vulkan/icd.d/nvidia_icd.json + +# Setup the required capabilities for the container runtime +ENV NVIDIA_DRIVER_CAPABILITIES compute,graphics,utility diff --git a/Util/Docker/vulkan-base/nvidia_icd.json b/Util/Docker/vulkan-base/nvidia_icd.json new file mode 100644 index 00000000000..b64452bf924 --- /dev/null +++ b/Util/Docker/vulkan-base/nvidia_icd.json @@ -0,0 +1,7 @@ +{ + "file_format_version" : "1.0.0", + "ICD": { + "library_path": "libGLX_nvidia.so.0", + "api_version" : "1.3.268" + } +} From 0f78bab47620575a8fd9cac1570bb18ada1a5ab7 Mon Sep 17 00:00:00 2001 From: Benedikt Haas Date: Fri, 5 Jan 2024 15:50:15 +0100 Subject: [PATCH 63/94] use new base image in release Dockerfile --- Util/Docker/Release.Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Util/Docker/Release.Dockerfile b/Util/Docker/Release.Dockerfile index 39d4b6d47a9..0b19403ff5e 100644 --- a/Util/Docker/Release.Dockerfile +++ b/Util/Docker/Release.Dockerfile @@ -3,9 +3,7 @@ # # sudo -E docker run --rm --gpus all -it --net=host carla:latest /bin/bash -FROM nvidia/vulkan:1.1.121-cuda-10.1--ubuntu18.04 - -RUN apt-key adv --fetch-keys "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub" +FROM vulkan-base:latest RUN packages='libsdl2-2.0 xserver-xorg libvulkan1 libomp5' && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y $packages --no-install-recommends From c8823bbcfb8e0466494d7c5c8576408e68bc13d9 Mon Sep 17 00:00:00 2001 From: Benedikt Haas Date: Fri, 5 Jan 2024 15:58:12 +0100 Subject: [PATCH 64/94] integrate building vulkan-base into CI pipeline --- .github/workflows/docker.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f4688df252d..fe80439bb61 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -95,12 +95,29 @@ jobs: commit: ${{ github.sha }} artifacts: artifacts/PythonAPI.tar.gz token: ${{ secrets.PAT }} + + # create vulkan-base for carla release + create-vulkan-base-image: + name: Create vulkan-base image + runs-on: self-hosted + needs: clean-up + steps: + + - uses: docker/build-push-action@v5 + name: Build image + with: + file: Util/Docker/vulkan-base/Vulkan.Dockerfile + tags: vulkan-base + no-cache: true + build-args: | + BASE_DIST=ubuntu22.04 + CUDA_VERSION=12.2.0 # create and push carla image create-carla-image: name: Create carla image runs-on: self-hosted - needs: provide-artifacts + needs: [provide-artifacts, create-vulkan-base-image] steps: - uses: docker/login-action@v3 From 97921bd4efe591a45be710279daa51d54fbc7f0e Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Fri, 5 Jan 2024 16:28:05 +0100 Subject: [PATCH 65/94] set build context for vulkan-base-image --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fe80439bb61..e4bf432b1f4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -112,6 +112,7 @@ jobs: build-args: | BASE_DIST=ubuntu22.04 CUDA_VERSION=12.2.0 + context: Util/Docker/vulkan-base # create and push carla image create-carla-image: From 27ff5c88fb80b1f8990d49eb5036470e16300c9d Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Fri, 5 Jan 2024 16:42:02 +0100 Subject: [PATCH 66/94] remove job parallalization --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e4bf432b1f4..be40c6505a8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,7 +17,7 @@ jobs: create-carla-prerequisites-image: name: Create carla-prerequisites image runs-on: self-hosted - needs: clean-up + needs: create-vulkan-base-image steps: - uses: docker/build-push-action@v5 From b0984de56ea5463860c3fa92a16dd50133ddbaaf Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Fri, 5 Jan 2024 16:51:34 +0100 Subject: [PATCH 67/94] add checkout action to vulkan-base job --- .github/workflows/docker.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index be40c6505a8..3c1190ae4a9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -102,7 +102,11 @@ jobs: runs-on: self-hosted needs: clean-up steps: - + - uses: actions/checkout@v3 + name: Checkout repository + with: + submodules: true + - uses: docker/build-push-action@v5 name: Build image with: From 1ca0a1136f390a94d2d168fcd2eeb02106551f58 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Fri, 5 Jan 2024 17:08:30 +0100 Subject: [PATCH 68/94] shift vukan-base stage --- .github/workflows/docker.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3c1190ae4a9..a1798e59996 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,7 +17,7 @@ jobs: create-carla-prerequisites-image: name: Create carla-prerequisites image runs-on: self-hosted - needs: create-vulkan-base-image + needs: clean-up steps: - uses: docker/build-push-action@v5 @@ -100,13 +100,14 @@ jobs: create-vulkan-base-image: name: Create vulkan-base image runs-on: self-hosted - needs: clean-up + needs: provide-artifacts steps: + - uses: actions/checkout@v3 name: Checkout repository with: submodules: true - + - uses: docker/build-push-action@v5 name: Build image with: @@ -122,7 +123,7 @@ jobs: create-carla-image: name: Create carla image runs-on: self-hosted - needs: [provide-artifacts, create-vulkan-base-image] + needs: [create-vulkan-base-image] steps: - uses: docker/login-action@v3 From 973eddc66e8bc2d5fc2355e49b9959d6f6e9c89e Mon Sep 17 00:00:00 2001 From: Benedikt Haas Date: Mon, 8 Jan 2024 11:24:14 +0100 Subject: [PATCH 69/94] remove checkout before vulkan-base build --- .github/workflows/docker.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a1798e59996..377d8d7c3f9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -102,11 +102,6 @@ jobs: runs-on: self-hosted needs: provide-artifacts steps: - - - uses: actions/checkout@v3 - name: Checkout repository - with: - submodules: true - uses: docker/build-push-action@v5 name: Build image From 7b7d1a6ae51ebdf9b338d47d36f014806d38f40b Mon Sep 17 00:00:00 2001 From: Benedikt Haas Date: Mon, 8 Jan 2024 15:39:31 +0100 Subject: [PATCH 70/94] fix append to pythonpath --- PythonAPI/util/ping.py | 1 + 1 file changed, 1 insertion(+) diff --git a/PythonAPI/util/ping.py b/PythonAPI/util/ping.py index 158d1877b9c..dac97d682ad 100644 --- a/PythonAPI/util/ping.py +++ b/PythonAPI/util/ping.py @@ -13,6 +13,7 @@ import sys try: + os.chdir(os.path.dirname(__file__)) sys.path.append(glob.glob('../carla/dist/carla-*%d.%d-%s.egg' % ( sys.version_info.major, sys.version_info.minor, From dc75ccdc2d7c5b61facc88e301ab2d977ff96b37 Mon Sep 17 00:00:00 2001 From: Benedikt Haas Date: Mon, 8 Jan 2024 15:42:29 +0100 Subject: [PATCH 71/94] autogenerate driver conf and pythonpath env setup script --- Util/Docker/Release.Dockerfile | 9 +++++++++ Util/Docker/vulkan-base/Vulkan.Dockerfile | 18 ++++++++++++------ Util/Docker/vulkan-base/nvidia_icd.json | 7 ------- 3 files changed, 21 insertions(+), 13 deletions(-) delete mode 100644 Util/Docker/vulkan-base/nvidia_icd.json diff --git a/Util/Docker/Release.Dockerfile b/Util/Docker/Release.Dockerfile index 0b19403ff5e..cc926069e93 100644 --- a/Util/Docker/Release.Dockerfile +++ b/Util/Docker/Release.Dockerfile @@ -14,6 +14,15 @@ COPY --chown=carla:carla . /home/carla USER carla WORKDIR /home/carla +# Create env setup script to make CARLA PythonAPI easily accessible (needs interactive shell!) +ARG CARLA_PYAPI_PATH=/home/carla/PythonAPI +ENV CARLA_PYAPI_PATH=${CARLA_PYAPI_PATH} + +RUN echo "export PYTHONPATH=\$PYTHONPATH:$CARLA_PYAPI_PATH/carla/dist/$(ls $CARLA_PYAPI_PATH/carla/dist | grep .egg)" >> $CARLA_PYAPI_PATH/setup_env.sh; \ + echo "export PYTHONPATH=\$PYTHONPATH:$CARLA_PYAPI_PATH/carla/agents" >> $CARLA_PYAPI_PATH/setup_env.sh; \ + echo "export PYTHONPATH=\$PYTHONPATH:$CARLA_PYAPI_PATH/carla" >> $CARLA_PYAPI_PATH/setup_env.sh; \ + echo "source $CARLA_PYAPI_PATH/setup_env.sh" >> /etc/bash.bashrc + # you can also run CARLA in offscreen mode with -RenderOffScreen # CMD /bin/bash CarlaUE4.sh -RenderOffScreen CMD /bin/bash CarlaUE4.sh diff --git a/Util/Docker/vulkan-base/Vulkan.Dockerfile b/Util/Docker/vulkan-base/Vulkan.Dockerfile index 793e1b79688..9c6455a9043 100644 --- a/Util/Docker/vulkan-base/Vulkan.Dockerfile +++ b/Util/Docker/vulkan-base/Vulkan.Dockerfile @@ -1,4 +1,4 @@ -# custom Dockerfile for the release base image (based off of discontinued https://gitlab.com/nvidia/container-images/vulkan ) +# Custom Dockerfile for the release base image (based off of discontinued https://gitlab.com/nvidia/container-images/vulkan ) ARG DEBIAN_FRONTEND=noninteractive ARG BASE_DIST=ubuntu22.04 @@ -20,13 +20,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # Install Vulkan SDK +# You can set VULKAN_SDK_VERSION as latest via build-arg=`curl https://vulkan.lunarg.com/sdk/latest/linux.txt` ARG VULKAN_SDK_VERSION=1.3.268.0 -#set VULKAN_SDK_VERSION as latest via build-arg=`curl https://vulkan.lunarg.com/sdk/latest/linux.txt` -ARG VULKAN_SDK_VERSION +ARG VULKAN_API_VERSION=1.3.268 # Download the Vulkan SDK and extract the headers, loaders, layers and binary utilities RUN wget -q --show-progress \ --progress=bar:force:noscroll \ - https://sdk.lunarg.com/sdk/download/latest/linux/vulkan_sdk.tar.gz \ + https://sdk.lunarg.com/sdk/download/${VULKAN_SDK_VERSION}/linux/vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.xz \ -O /tmp/vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.gz \ && echo "Installing Vulkan SDK ${VULKAN_SDK_VERSION}" \ && mkdir -p /opt/vulkan \ @@ -42,8 +42,14 @@ RUN wget -q --show-progress \ && ldconfig \ && rm /tmp/vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.gz && rm -rf /opt/vulkan -# Copy in Nvidia driver config -COPY nvidia_icd.json /etc/vulkan/icd.d/nvidia_icd.json +# Generate Nvidia driver config +RUN echo "{" > /etc/vulkan/icd.d/nvidia_icd.json; \ + echo " \"file_format_version\" : \"1.0.0\"," >> /etc/vulkan/icd.d/nvidia_icd.json; \ + echo " \"ICD\": {" >> /etc/vulkan/icd.d/nvidia_icd.json; \ + echo " \"library_path\": \"libGLX_nvidia.so.0\"," >> /etc/vulkan/icd.d/nvidia_icd.json; \ + echo " \"api_version\" : \"${VULKAN_API_VERSION}\"" >> /etc/vulkan/icd.d/nvidia_icd.json; \ + echo " }" >> /etc/vulkan/icd.d/nvidia_icd.json; \ + echo "}" >> /etc/vulkan/icd.d/nvidia_icd.json # Setup the required capabilities for the container runtime ENV NVIDIA_DRIVER_CAPABILITIES compute,graphics,utility diff --git a/Util/Docker/vulkan-base/nvidia_icd.json b/Util/Docker/vulkan-base/nvidia_icd.json deleted file mode 100644 index b64452bf924..00000000000 --- a/Util/Docker/vulkan-base/nvidia_icd.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "file_format_version" : "1.0.0", - "ICD": { - "library_path": "libGLX_nvidia.so.0", - "api_version" : "1.3.268" - } -} From bb67bec35f103fa0143bc93b255a7deda81f790f Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Mon, 8 Jan 2024 17:49:31 +0100 Subject: [PATCH 72/94] Using boost backup from aws carla-releases --- Util/BuildTools/Setup.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Util/BuildTools/Setup.sh b/Util/BuildTools/Setup.sh index 9aa65533bf0..26e2b3570a7 100755 --- a/Util/BuildTools/Setup.sh +++ b/Util/BuildTools/Setup.sh @@ -91,13 +91,11 @@ for PY_VERSION in ${PY_VERSION_LIST[@]} ; do BOOST_PACKAGE_BASENAME=boost_${BOOST_VERSION//./_} log "Retrieving boost." - wget "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/${BOOST_PACKAGE_BASENAME}.tar.gz" || true - # try to use the backup boost we have in Jenkins - if [[ ! -f "${BOOST_PACKAGE_BASENAME}.tar.gz" ]] ; then - log "Using boost backup" - wget "https://carla-releases.s3.eu-west-3.amazonaws.com/Backup/${BOOST_PACKAGE_BASENAME}.tar.gz" || true - fi - + + # use boost backup instead of official release due to required verification + log "Using boost backup from aws carla-releases" + wget "https://carla-releases.s3.eu-west-3.amazonaws.com/Backup/${BOOST_PACKAGE_BASENAME}.tar.gz" || true + log "Extracting boost for Python ${PY_VERSION}." tar -xzf ${BOOST_PACKAGE_BASENAME}.tar.gz mkdir -p ${BOOST_BASENAME}-install/include From 491a9588e524ea30a0cff63bff5b178512733f2a Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Mon, 8 Jan 2024 22:18:14 +0100 Subject: [PATCH 73/94] Update Vulkan.Dockerfile --- Util/Docker/vulkan-base/Vulkan.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Util/Docker/vulkan-base/Vulkan.Dockerfile b/Util/Docker/vulkan-base/Vulkan.Dockerfile index 9c6455a9043..6f1462d446d 100644 --- a/Util/Docker/vulkan-base/Vulkan.Dockerfile +++ b/Util/Docker/vulkan-base/Vulkan.Dockerfile @@ -43,7 +43,8 @@ RUN wget -q --show-progress \ && rm /tmp/vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.gz && rm -rf /opt/vulkan # Generate Nvidia driver config -RUN echo "{" > /etc/vulkan/icd.d/nvidia_icd.json; \ +RUN mkdir -p /etc/vulkan/icd.d && \ + echo "{" > /etc/vulkan/icd.d/nvidia_icd.json; \ echo " \"file_format_version\" : \"1.0.0\"," >> /etc/vulkan/icd.d/nvidia_icd.json; \ echo " \"ICD\": {" >> /etc/vulkan/icd.d/nvidia_icd.json; \ echo " \"library_path\": \"libGLX_nvidia.so.0\"," >> /etc/vulkan/icd.d/nvidia_icd.json; \ From 49cdf42116decd9dde8494d5b227228af3432abb Mon Sep 17 00:00:00 2001 From: Benedikt Haas Date: Tue, 9 Jan 2024 13:18:35 +0100 Subject: [PATCH 74/94] fix carla user permission denied issue --- Util/Docker/Release.Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Util/Docker/Release.Dockerfile b/Util/Docker/Release.Dockerfile index cc926069e93..46b7d99ca7e 100644 --- a/Util/Docker/Release.Dockerfile +++ b/Util/Docker/Release.Dockerfile @@ -11,9 +11,6 @@ RUN useradd -m carla COPY --chown=carla:carla . /home/carla -USER carla -WORKDIR /home/carla - # Create env setup script to make CARLA PythonAPI easily accessible (needs interactive shell!) ARG CARLA_PYAPI_PATH=/home/carla/PythonAPI ENV CARLA_PYAPI_PATH=${CARLA_PYAPI_PATH} @@ -23,6 +20,9 @@ RUN echo "export PYTHONPATH=\$PYTHONPATH:$CARLA_PYAPI_PATH/carla/dist/$(ls $CARL echo "export PYTHONPATH=\$PYTHONPATH:$CARLA_PYAPI_PATH/carla" >> $CARLA_PYAPI_PATH/setup_env.sh; \ echo "source $CARLA_PYAPI_PATH/setup_env.sh" >> /etc/bash.bashrc +USER carla +WORKDIR /home/carla + # you can also run CARLA in offscreen mode with -RenderOffScreen # CMD /bin/bash CarlaUE4.sh -RenderOffScreen CMD /bin/bash CarlaUE4.sh From ade5046ef3d19545a9f63e47324e7dadbc8e65c9 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Thu, 25 Jan 2024 09:15:26 +0100 Subject: [PATCH 75/94] use ubuntu-latest instead of self-hosted --- .github/workflows/docker.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 377d8d7c3f9..f9910e06d89 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,7 +7,7 @@ jobs: # clean up clean-up: name: Clean up - runs-on: self-hosted + runs-on: ubuntu-latest steps: - name: Clean up @@ -16,7 +16,7 @@ jobs: # create carla-prerequisites image create-carla-prerequisites-image: name: Create carla-prerequisites image - runs-on: self-hosted + runs-on: ubuntu-latest needs: clean-up steps: @@ -33,7 +33,7 @@ jobs: # create and push carla-source image create-carla-source-image: name: Create carla-source image - runs-on: self-hosted + runs-on: ubuntu-latest needs: create-carla-prerequisites-image steps: - uses: actions/checkout@v3 @@ -53,13 +53,13 @@ jobs: file: Util/Docker/Carla.Dockerfile tags: cgellerac/carla-simulator:source no-cache: true - push: true + push: true # TODO set to false for rwthika context: . # provide artifacts and releases provide-artifacts: name: Provide carla artifacts - runs-on: self-hosted + runs-on: ubuntu-latest needs: create-carla-source-image steps: @@ -99,7 +99,7 @@ jobs: # create vulkan-base for carla release create-vulkan-base-image: name: Create vulkan-base image - runs-on: self-hosted + runs-on: ubuntu-latest needs: provide-artifacts steps: @@ -117,8 +117,8 @@ jobs: # create and push carla image create-carla-image: name: Create carla image - runs-on: self-hosted - needs: [create-vulkan-base-image] + runs-on: ubuntu-latest + needs: create-vulkan-base-image steps: - uses: docker/login-action@v3 From f8c84ba7d762dafd05b215f2832d48c18b8806e7 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Thu, 25 Jan 2024 10:08:46 +0100 Subject: [PATCH 76/94] reset to self-hosted runner due to memory --- .github/workflows/docker.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f9910e06d89..c30823c0151 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,7 +7,7 @@ jobs: # clean up clean-up: name: Clean up - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Clean up @@ -16,7 +16,7 @@ jobs: # create carla-prerequisites image create-carla-prerequisites-image: name: Create carla-prerequisites image - runs-on: ubuntu-latest + runs-on: self-hosted needs: clean-up steps: @@ -33,7 +33,7 @@ jobs: # create and push carla-source image create-carla-source-image: name: Create carla-source image - runs-on: ubuntu-latest + runs-on: self-hosted needs: create-carla-prerequisites-image steps: - uses: actions/checkout@v3 @@ -59,7 +59,7 @@ jobs: # provide artifacts and releases provide-artifacts: name: Provide carla artifacts - runs-on: ubuntu-latest + runs-on: self-hosted needs: create-carla-source-image steps: @@ -99,7 +99,7 @@ jobs: # create vulkan-base for carla release create-vulkan-base-image: name: Create vulkan-base image - runs-on: ubuntu-latest + runs-on: self-hosted needs: provide-artifacts steps: @@ -117,7 +117,7 @@ jobs: # create and push carla image create-carla-image: name: Create carla image - runs-on: ubuntu-latest + runs-on: self-hosted needs: create-vulkan-base-image steps: From 0a10db60c442b95123125d865073abf1e409ff98 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Sat, 27 Jan 2024 11:31:50 +0100 Subject: [PATCH 77/94] Update dockerhub remote --- .github/workflows/docker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c30823c0151..bc790354d30 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -51,7 +51,7 @@ jobs: name: Build and push image with: file: Util/Docker/Carla.Dockerfile - tags: cgellerac/carla-simulator:source + tags: rwthika/carla-simulator:source no-cache: true push: true # TODO set to false for rwthika context: . @@ -67,7 +67,7 @@ jobs: - uses: shrink/actions-docker-extract@v3 name: Extract carla-package with: - image: cgellerac/carla-simulator:source + image: rwthika/carla-simulator:source path: home/carla/carla/Dist destination: artifacts/ @@ -76,7 +76,7 @@ jobs: name: Extract carla-python-api id: extract_python_api with: - image: cgellerac/carla-simulator:source + image: rwthika/carla-simulator:source path: home/carla/carla/PythonAPI destination: artifacts/ @@ -134,7 +134,7 @@ jobs: name: Build and push with: file: build/Dockerfile - tags: cgellerac/carla-simulator + tags: rwthika/carla-simulator no-cache: true context: build push: true From c91b64a37e270288ea27d652571f90ad652184be Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Mon, 29 Jan 2024 12:23:56 +0100 Subject: [PATCH 78/94] Disable aria2 content download --- Update.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Update.sh b/Update.sh index 72ed6390727..cbd579f1fcd 100755 --- a/Update.sh +++ b/Update.sh @@ -58,13 +58,15 @@ function download_content { fi mkdir -p "$CONTENT_FOLDER" mkdir -p Content - if hash aria2c 2>/dev/null; then - echo -e "${CONTENT_LINK}\n\tout=Content.tar.gz" > .aria2c.input - aria2c -j16 -x16 --input-file=.aria2c.input - rm -f .aria2c.input - else + + #if hash aria2c 2>/dev/null; then + # echo -e "${CONTENT_LINK}\n\tout=Content.tar.gz" > .aria2c.input + # aria2c -j16 -x16 --input-file=.aria2c.input + # rm -f .aria2c.input + #else wget -c ${CONTENT_LINK} -O Content.tar.gz - fi + #fi + tar -xvzf Content.tar.gz -C Content rm Content.tar.gz mv Content/* "$CONTENT_FOLDER" From 21f29a77a459d44a81ff0f88e936f71a54a190e8 Mon Sep 17 00:00:00 2001 From: Benedikt Haas Date: Mon, 29 Jan 2024 13:50:13 +0100 Subject: [PATCH 79/94] configure healthcheck --- PythonAPI/util/ping.py | 0 Util/Docker/Release.Dockerfile | 3 +++ 2 files changed, 3 insertions(+) mode change 100644 => 100755 PythonAPI/util/ping.py diff --git a/PythonAPI/util/ping.py b/PythonAPI/util/ping.py old mode 100644 new mode 100755 diff --git a/Util/Docker/Release.Dockerfile b/Util/Docker/Release.Dockerfile index 46b7d99ca7e..0723f184973 100644 --- a/Util/Docker/Release.Dockerfile +++ b/Util/Docker/Release.Dockerfile @@ -23,6 +23,9 @@ RUN echo "export PYTHONPATH=\$PYTHONPATH:$CARLA_PYAPI_PATH/carla/dist/$(ls $CARL USER carla WORKDIR /home/carla +HEALTHCHECK --interval=5s --timeout=5s --start-period=10s --retries=3 \ + CMD python3 ./PythonAPI/util/ping.py + # you can also run CARLA in offscreen mode with -RenderOffScreen # CMD /bin/bash CarlaUE4.sh -RenderOffScreen CMD /bin/bash CarlaUE4.sh From 29b662c69ecc34bb3f416e96cf1a295c55e66e58 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Mon, 29 Jan 2024 14:46:31 +0100 Subject: [PATCH 80/94] update documentation --- FEATURES.md | 13 +++++++++++++ README.md | 15 +++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 FEATURES.md diff --git a/FEATURES.md b/FEATURES.md new file mode 100644 index 00000000000..63ffb65a339 --- /dev/null +++ b/FEATURES.md @@ -0,0 +1,13 @@ +# Feature List +All changes are modifications to the original repository. + +## Core Features +- GitHub workflow to automatically build Docker images +- Update to Ubuntu 20.04 and Python3.10 for build image +- Update to Ubuntu 22.04 and Python3.10 for release image +- ping.py script to check availability of the carla-simulator + +## Minor Features +- use wget instead of aria2 for content download +- use private copy of fbxsdk +- use boost library from carla aws backup storage \ No newline at end of file diff --git a/README.md b/README.md index a596c830622..6b3f003ff17 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,22 @@ # Simulation Core: *carla-simulator* -

- +

+ - + + +

-We recommend to use the *carla-simulator* as **simulation core** within our proposed simulation framework carla-compose. +> [!IMPORTANT] +> This repository is a minimal fork of the official [carla-simulator](https://github.com/carla-simulator/carla)! All modifications to the original repository are documented in [FEATURES.md](./FEATURES.md). +> [!RECOMMENDATION] +> We recommend to use the *carla-simulator* as **simulation core** in our open, modular and scalable simulation framework **CARLOS**. Here, it constitutes the central element of a simulation and handles all graphical and dynamic calculations in the individual simulation time steps. + +**Note**: We set up a Continous Integration (CI) pipeline as [GitHub action](./github/workflows/docker.yml) to continously build Docker images for the `carla-simulator`, publicly available on [Docker Hub](https://hub.docker.com/r/rwthika/carla-simulator). --- --- From 5efe7ac5544833f3d595c58dac83d725bf3e6b28 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Mon, 29 Jan 2024 14:53:13 +0100 Subject: [PATCH 81/94] Update readme --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6b3f003ff17..c10f984647a 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,12 @@ > [!IMPORTANT] > This repository is a minimal fork of the official [carla-simulator](https://github.com/carla-simulator/carla)! All modifications to the original repository are documented in [FEATURES.md](./FEATURES.md). -> [!RECOMMENDATION] -> We recommend to use the *carla-simulator* as **simulation core** in our open, modular and scalable simulation framework **CARLOS**. Here, it constitutes the central element of a simulation and handles all graphical and dynamic calculations in the individual simulation time steps. +> [!TIP] +> We recommend to use the *carla-simulator* as **simulation core** in our open, modular and scalable simulation framework **CARLOS**. +> Here, it constitutes the central element of a simulation and handles all graphical and dynamic calculations in the individual simulation time steps. -**Note**: We set up a Continous Integration (CI) pipeline as [GitHub action](./github/workflows/docker.yml) to continously build Docker images for the `carla-simulator`, publicly available on [Docker Hub](https://hub.docker.com/r/rwthika/carla-simulator). +> [!NOTE] +> We set up a Continous Integration (CI) pipeline as [GitHub action](./github/workflows/docker.yml) to continously build Docker images for the `carla-simulator`, publicly available on [Docker Hub](https://hub.docker.com/r/rwthika/carla-simulator). --- --- From e3714a691cb8e667483929aa6404bf73e2ce268e Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Mon, 29 Jan 2024 15:11:09 +0100 Subject: [PATCH 82/94] use ika-rwth-aachen url --- FEATURES.md | 7 ++++--- README.md | 11 ++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/FEATURES.md b/FEATURES.md index 63ffb65a339..dc6a1c97450 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -2,10 +2,11 @@ All changes are modifications to the original repository. ## Core Features -- GitHub workflow to automatically build Docker images -- Update to Ubuntu 20.04 and Python3.10 for build image -- Update to Ubuntu 22.04 and Python3.10 for release image +- add GitHub workflow to automatically build Docker images +- update to Ubuntu 20.04 and Python3.10 for build image +- update to Ubuntu 22.04 and Python3.10 for release image - ping.py script to check availability of the carla-simulator +- integrate and provide Docker health check ## Minor Features - use wget instead of aria2 for content download diff --git a/README.md b/README.md index c10f984647a..60bce073a51 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # Simulation Core: *carla-simulator*

- - - - + + + + - +

> [!IMPORTANT] @@ -15,6 +15,7 @@ > [!TIP] > We recommend to use the *carla-simulator* as **simulation core** in our open, modular and scalable simulation framework **CARLOS**. +> > Here, it constitutes the central element of a simulation and handles all graphical and dynamic calculations in the individual simulation time steps. > [!NOTE] From dc2219ca56725d61f095402dcbe0e125063270cd Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Mon, 29 Jan 2024 17:07:23 +0100 Subject: [PATCH 83/94] add image tag handling --- .github/workflows/docker.yml | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index bc790354d30..b8b06feab16 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,16 +2,25 @@ name: Docker on: [push, pull_request] +env: + IMAGE_TAG_SUFFIX: "" + jobs: - # clean up - clean-up: - name: Clean up + # set up + set-up: + name: Set up runs-on: self-hosted steps: - name: Clean up run: rm -rf * + + - name: Set image tag + run: | + echo "IMAGE_TAG_SUFFIX=_${{ github.ref_name }}_ci" >> $GITHUB_ENV + if: github.ref.name != github.event.repository.default_branch + # create carla-prerequisites image create-carla-prerequisites-image: @@ -51,9 +60,9 @@ jobs: name: Build and push image with: file: Util/Docker/Carla.Dockerfile - tags: rwthika/carla-simulator:source + tags: rwthika/carla-simulator:source${{env.IMAGE_TAG_SUFFIX}} no-cache: true - push: true # TODO set to false for rwthika + push: true context: . # provide artifacts and releases @@ -62,12 +71,12 @@ jobs: runs-on: self-hosted needs: create-carla-source-image steps: - + # provide carla-package - uses: shrink/actions-docker-extract@v3 name: Extract carla-package with: - image: rwthika/carla-simulator:source + image: rwthika/carla-simulator:source${{env.IMAGE_TAG_SUFFIX}} path: home/carla/carla/Dist destination: artifacts/ @@ -76,7 +85,7 @@ jobs: name: Extract carla-python-api id: extract_python_api with: - image: rwthika/carla-simulator:source + image: rwthika/carla-simulator:source${{env.IMAGE_TAG_SUFFIX}} path: home/carla/carla/PythonAPI destination: artifacts/ @@ -134,7 +143,7 @@ jobs: name: Build and push with: file: build/Dockerfile - tags: rwthika/carla-simulator + tags: rwthika/carla-simulator:latest${{env.IMAGE_TAG_SUFFIX}} no-cache: true context: build push: true From 3faf211299151ba06c5d60afcaa21b217840cfb4 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Mon, 29 Jan 2024 17:23:39 +0100 Subject: [PATCH 84/94] replace / characters --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b8b06feab16..963e7210b2d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -18,7 +18,7 @@ jobs: - name: Set image tag run: | - echo "IMAGE_TAG_SUFFIX=_${{ github.ref_name }}_ci" >> $GITHUB_ENV + echo "IMAGE_TAG_SUFFIX=_$(echo '${{ github.ref_name }}' | sed '/\//-/g')_ci" >> $GITHUB_ENV if: github.ref.name != github.event.repository.default_branch From 1158ea99e9def1eff88ef87c028d8123b0923bc8 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Mon, 29 Jan 2024 17:50:19 +0100 Subject: [PATCH 85/94] use tr instead of sed --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 963e7210b2d..20c40cccede 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -18,7 +18,7 @@ jobs: - name: Set image tag run: | - echo "IMAGE_TAG_SUFFIX=_$(echo '${{ github.ref_name }}' | sed '/\//-/g')_ci" >> $GITHUB_ENV + echo "IMAGE_TAG_SUFFIX=_$(echo '${{ github.ref_name }}' | tr / -)_ci" >> $GITHUB_ENV if: github.ref.name != github.event.repository.default_branch From 2ce3a1a5651bf2ff1cd0695de20436293a6c6b81 Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Mon, 29 Jan 2024 18:04:47 +0100 Subject: [PATCH 86/94] disable source image push --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 20c40cccede..3b0aae54e33 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -62,7 +62,7 @@ jobs: file: Util/Docker/Carla.Dockerfile tags: rwthika/carla-simulator:source${{env.IMAGE_TAG_SUFFIX}} no-cache: true - push: true + push: false context: . # provide artifacts and releases From 5d40d8a52caadd06ab633a966397182c42d17d5d Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Mon, 29 Jan 2024 18:16:34 +0100 Subject: [PATCH 87/94] update depends action --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3b0aae54e33..4855364fac9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -26,7 +26,7 @@ jobs: create-carla-prerequisites-image: name: Create carla-prerequisites image runs-on: self-hosted - needs: clean-up + needs: set-up steps: - uses: docker/build-push-action@v5 From 1aab781980155772eb5b2783720b45a724d43949 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Tue, 30 Jan 2024 00:42:52 +0100 Subject: [PATCH 88/94] Use local envs for IMAGE_TAG_SUFFIX --- .github/workflows/docker.yml | 39 +++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4855364fac9..a4260e1e4aa 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,31 +2,22 @@ name: Docker on: [push, pull_request] -env: - IMAGE_TAG_SUFFIX: "" - jobs: # set up - set-up: - name: Set up + clean-up: + name: Clean up runs-on: self-hosted steps: - name: Clean up run: rm -rf * - - - name: Set image tag - run: | - echo "IMAGE_TAG_SUFFIX=_$(echo '${{ github.ref_name }}' | tr / -)_ci" >> $GITHUB_ENV - if: github.ref.name != github.event.repository.default_branch - # create carla-prerequisites image create-carla-prerequisites-image: name: Create carla-prerequisites image runs-on: self-hosted - needs: set-up + needs: clean-up steps: - uses: docker/build-push-action@v5 @@ -44,7 +35,16 @@ jobs: name: Create carla-source image runs-on: self-hosted needs: create-carla-prerequisites-image + env: + IMAGE_TAG_SUFFIX: "" + steps: + + - name: Set image tag + run: | + echo "IMAGE_TAG_SUFFIX=_$(echo '${{ github.ref_name }}' | tr / -)_ci" >> $GITHUB_ENV + if: github.ref.name != github.event.repository.default_branch + - uses: actions/checkout@v3 name: Checkout repository with: @@ -70,7 +70,15 @@ jobs: name: Provide carla artifacts runs-on: self-hosted needs: create-carla-source-image + env: + IMAGE_TAG_SUFFIX: "" + steps: + + - name: Set image tag + run: | + echo "IMAGE_TAG_SUFFIX=_$(echo '${{ github.ref_name }}' | tr / -)_ci" >> $GITHUB_ENV + if: github.ref.name != github.event.repository.default_branch # provide carla-package - uses: shrink/actions-docker-extract@v3 @@ -128,7 +136,14 @@ jobs: name: Create carla image runs-on: self-hosted needs: create-vulkan-base-image + env: + IMAGE_TAG_SUFFIX: "" + steps: + - name: Set image tag + run: | + echo "IMAGE_TAG_SUFFIX=_$(echo '${{ github.ref_name }}' | tr / -)_ci" >> $GITHUB_ENV + if: github.ref.name != github.event.repository.default_branch - uses: docker/login-action@v3 name: Login to Docker Hub From 3ab2cd81222009dbb50b83e0dd4620557d19568c Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Tue, 30 Jan 2024 08:58:55 +0100 Subject: [PATCH 89/94] Update docker.yml fix set image tag steps --- .github/workflows/docker.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a4260e1e4aa..48d85da91cc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -40,7 +40,7 @@ jobs: steps: - - name: Set image tag + - name: Set image tag run: | echo "IMAGE_TAG_SUFFIX=_$(echo '${{ github.ref_name }}' | tr / -)_ci" >> $GITHUB_ENV if: github.ref.name != github.event.repository.default_branch @@ -75,7 +75,7 @@ jobs: steps: - - name: Set image tag + - name: Set image tag run: | echo "IMAGE_TAG_SUFFIX=_$(echo '${{ github.ref_name }}' | tr / -)_ci" >> $GITHUB_ENV if: github.ref.name != github.event.repository.default_branch @@ -140,7 +140,8 @@ jobs: IMAGE_TAG_SUFFIX: "" steps: - - name: Set image tag + + - name: Set image tag run: | echo "IMAGE_TAG_SUFFIX=_$(echo '${{ github.ref_name }}' | tr / -)_ci" >> $GITHUB_ENV if: github.ref.name != github.event.repository.default_branch From b0a10c54356fa9db651d88fb57f05baa011b8e9a Mon Sep 17 00:00:00 2001 From: Christian Geller Date: Tue, 30 Jan 2024 14:24:00 +0100 Subject: [PATCH 90/94] revert all fbxsdk changes --- Util/BuildTools/BuildUtilsDocker.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Util/BuildTools/BuildUtilsDocker.sh b/Util/BuildTools/BuildUtilsDocker.sh index c1aa6f3de8c..2c85a7b053c 100755 --- a/Util/BuildTools/BuildUtilsDocker.sh +++ b/Util/BuildTools/BuildUtilsDocker.sh @@ -12,18 +12,22 @@ if [ -f "${FBX2OBJ_DIST}/FBX2OBJ" ]; then exit fi -LIB_NAME=fbx202034_fbxsdk_linux -FBXSDK_URL=https://rwth-aachen.sciebo.de/s/z4KgTOD39v4I8Qw/download # Private Sciebo +LIB_NAME=fbx202001_fbxsdk_linux +FBXSDK_URL=https://www.autodesk.com/content/dam/autodesk/www/adn/fbx/2020-0-1/${LIB_NAME}.tar.gz if [ ! -d "${FBX2OBJ_DEP_FOLDER}" ]; then log "Downloading FBX SDK..." - wget -O "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}.zip" "${FBXSDK_URL}" + wget -c "${FBXSDK_URL}" -P "${CARLA_DOCKER_UTILS_FOLDER}" --user-agent="Mozilla" echo "Unpacking..." mkdir -p "${FBX2OBJ_DEP_FOLDER}" - unzip "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}.zip" -d "${FBX2OBJ_DEP_FOLDER}" - rm "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}.zip" + tar -xvzf "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}.tar.gz" -C "${CARLA_DOCKER_UTILS_FOLDER}" "${LIB_NAME}" + rm "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}.tar.gz" + echo "Installing FBX SDK..." + echo -e "y\nyes\nn\n" | "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}" "${FBX2OBJ_DEP_FOLDER}" + echo + rm "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}" fi log "Compiling FBX2OBJ..." From 672395b55e1f1117c03b13799128d0403312ec72 Mon Sep 17 00:00:00 2001 From: Benedikt Haas Date: Tue, 30 Jan 2024 16:23:29 +0100 Subject: [PATCH 91/94] apply relevant review points --- CARLOS_CHANGELOG.md | 16 ++++++++++++++++ FEATURES.md | 14 -------------- README.md | 4 ++-- Util/BuildTools/Setup.sh | 10 ++++++---- Util/Docker/Release.Dockerfile | 2 +- 5 files changed, 25 insertions(+), 21 deletions(-) create mode 100644 CARLOS_CHANGELOG.md delete mode 100644 FEATURES.md diff --git a/CARLOS_CHANGELOG.md b/CARLOS_CHANGELOG.md new file mode 100644 index 00000000000..b1bf024233e --- /dev/null +++ b/CARLOS_CHANGELOG.md @@ -0,0 +1,16 @@ +## Latest : 1.0.0 + +## 1.0.0 - Initial Release + +### Major changes + +* Created GitHub workflow and Dockerfile to automatically build Docker images +* Update to Ubuntu 22.04 and Python 3.10 for prerequisites image +* Update to Ubuntu 22.04 and Python 3.10 for release image +* Created `vulkan-base` Dockerfile to replace discontinued `nvidia/vulkan` images used for the release image +* Added health check to release image + +### Minor changes + +* Added `ping.py` script to PythonAPI to enable health checks +* Fix [aria2](https://aria2.github.io/) related issue diff --git a/FEATURES.md b/FEATURES.md deleted file mode 100644 index dc6a1c97450..00000000000 --- a/FEATURES.md +++ /dev/null @@ -1,14 +0,0 @@ -# Feature List -All changes are modifications to the original repository. - -## Core Features -- add GitHub workflow to automatically build Docker images -- update to Ubuntu 20.04 and Python3.10 for build image -- update to Ubuntu 22.04 and Python3.10 for release image -- ping.py script to check availability of the carla-simulator -- integrate and provide Docker health check - -## Minor Features -- use wget instead of aria2 for content download -- use private copy of fbxsdk -- use boost library from carla aws backup storage \ No newline at end of file diff --git a/README.md b/README.md index 60bce073a51..b9725852b04 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@

> [!IMPORTANT] -> This repository is a minimal fork of the official [carla-simulator](https://github.com/carla-simulator/carla)! All modifications to the original repository are documented in [FEATURES.md](./FEATURES.md). +> This repository is a minimal fork of the official [carla-simulator](https://github.com/carla-simulator/carla)! All modifications to the original repository are documented in [CARLOS_CHANGELOG.md](./CARLOS_CHANGELOG.md). > [!TIP] > We recommend to use the *carla-simulator* as **simulation core** in our open, modular and scalable simulation framework **CARLOS**. @@ -19,7 +19,7 @@ > Here, it constitutes the central element of a simulation and handles all graphical and dynamic calculations in the individual simulation time steps. > [!NOTE] -> We set up a Continous Integration (CI) pipeline as [GitHub action](./github/workflows/docker.yml) to continously build Docker images for the `carla-simulator`, publicly available on [Docker Hub](https://hub.docker.com/r/rwthika/carla-simulator). +> We set up a Continous Integration (CI) pipeline as [GitHub workflow](./github/workflows/docker.yml) to continously build Docker images for the `carla-simulator`, publicly available on [Docker Hub](https://hub.docker.com/r/rwthika/carla-simulator). --- --- diff --git a/Util/BuildTools/Setup.sh b/Util/BuildTools/Setup.sh index 26e2b3570a7..d5827efea90 100755 --- a/Util/BuildTools/Setup.sh +++ b/Util/BuildTools/Setup.sh @@ -91,10 +91,12 @@ for PY_VERSION in ${PY_VERSION_LIST[@]} ; do BOOST_PACKAGE_BASENAME=boost_${BOOST_VERSION//./_} log "Retrieving boost." - - # use boost backup instead of official release due to required verification - log "Using boost backup from aws carla-releases" - wget "https://carla-releases.s3.eu-west-3.amazonaws.com/Backup/${BOOST_PACKAGE_BASENAME}.tar.gz" || true + wget "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/${BOOST_PACKAGE_BASENAME}.tar.gz" || true + # try to use the backup boost we have in Jenkins + if [[ ! -f "${BOOST_PACKAGE_BASENAME}.tar.gz" ]] ; then + log "Using boost backup" + wget "https://carla-releases.s3.eu-west-3.amazonaws.com/Backup/${BOOST_PACKAGE_BASENAME}.tar.gz" || true + fi log "Extracting boost for Python ${PY_VERSION}." tar -xzf ${BOOST_PACKAGE_BASENAME}.tar.gz diff --git a/Util/Docker/Release.Dockerfile b/Util/Docker/Release.Dockerfile index 0723f184973..590ca15f165 100644 --- a/Util/Docker/Release.Dockerfile +++ b/Util/Docker/Release.Dockerfile @@ -23,7 +23,7 @@ RUN echo "export PYTHONPATH=\$PYTHONPATH:$CARLA_PYAPI_PATH/carla/dist/$(ls $CARL USER carla WORKDIR /home/carla -HEALTHCHECK --interval=5s --timeout=5s --start-period=10s --retries=3 \ +HEALTHCHECK --interval=1s --timeout=5s --start-period=10s --retries=3 \ CMD python3 ./PythonAPI/util/ping.py # you can also run CARLA in offscreen mode with -RenderOffScreen From 485c2e103ebe6fdc3c01144fbb2447c2a99629ff Mon Sep 17 00:00:00 2001 From: Benedikt Haas Date: Tue, 30 Jan 2024 16:26:27 +0100 Subject: [PATCH 92/94] remove whitespace --- Util/BuildTools/Setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Util/BuildTools/Setup.sh b/Util/BuildTools/Setup.sh index d5827efea90..9aa65533bf0 100755 --- a/Util/BuildTools/Setup.sh +++ b/Util/BuildTools/Setup.sh @@ -97,7 +97,7 @@ for PY_VERSION in ${PY_VERSION_LIST[@]} ; do log "Using boost backup" wget "https://carla-releases.s3.eu-west-3.amazonaws.com/Backup/${BOOST_PACKAGE_BASENAME}.tar.gz" || true fi - + log "Extracting boost for Python ${PY_VERSION}." tar -xzf ${BOOST_PACKAGE_BASENAME}.tar.gz mkdir -p ${BOOST_BASENAME}-install/include From 34e20202a29c19daab87c00b0a87d89acc1fcc92 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Wed, 31 Jan 2024 10:54:47 +0100 Subject: [PATCH 93/94] Use curl to download fbxsdk --- Util/BuildTools/BuildUtilsDocker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Util/BuildTools/BuildUtilsDocker.sh b/Util/BuildTools/BuildUtilsDocker.sh index 2c85a7b053c..6689d0bfb77 100755 --- a/Util/BuildTools/BuildUtilsDocker.sh +++ b/Util/BuildTools/BuildUtilsDocker.sh @@ -17,7 +17,7 @@ FBXSDK_URL=https://www.autodesk.com/content/dam/autodesk/www/adn/fbx/2020-0-1/${ if [ ! -d "${FBX2OBJ_DEP_FOLDER}" ]; then log "Downloading FBX SDK..." - wget -c "${FBXSDK_URL}" -P "${CARLA_DOCKER_UTILS_FOLDER}" --user-agent="Mozilla" + curl -L "${FBXSDK_URL}" -o "${CARLA_DOCKER_UTILS_FOLDER}/${LIB_NAME}.tar.gz" --user-agent "Mozilla" echo "Unpacking..." mkdir -p "${FBX2OBJ_DEP_FOLDER}" From 2169bffd72ea743564ec7d34df1bb21b6f8d7c55 Mon Sep 17 00:00:00 2001 From: Christian Geller <88664444+cgeller@users.noreply.github.com> Date: Wed, 31 Jan 2024 10:56:26 +0100 Subject: [PATCH 94/94] Add fbx sdk curl change to changelog --- CARLOS_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CARLOS_CHANGELOG.md b/CARLOS_CHANGELOG.md index b1bf024233e..8bd7c3b2dbe 100644 --- a/CARLOS_CHANGELOG.md +++ b/CARLOS_CHANGELOG.md @@ -14,3 +14,4 @@ * Added `ping.py` script to PythonAPI to enable health checks * Fix [aria2](https://aria2.github.io/) related issue +* Fix fbx sdk download by using `curl` instead of `wget`