Merge pull request #59 from ferrandi/dev/panda

Dev/panda

.devcontainer/Dockerfile

@@ -17,7 +17,8 @@ ARG COMPILERS=gcc-4.9,clang-4
 COPY library-scripts/*.sh library-scripts/*.env /tmp/library-scripts/
 WORKDIR /
 RUN bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true" \
-    && bash /tmp/library-scripts/compiler-setup.sh "${COMPILERS}" \
+    && bash /tmp/library-scripts/compiler-download.sh "/" "${COMPILERS}" \
+    && bash /tmp/library-scripts/compiler-setup.sh \
     && apt-get install --no-install-recommends -y clang-format python3-defusedxml python3-distutils \
     && apt-get clean -y && rm -rf /var/lib/apt/lists/* /tmp/library-scripts
 

.devcontainer/Dockerfile.yosys

@@ -14,6 +14,7 @@ ARG COMPILERS=gcc-4.9,clang-4
 COPY library-scripts/*.sh library-scripts/*.env /tmp/library-scripts/
 WORKDIR /
 RUN bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true" \
-    && bash /tmp/library-scripts/compiler-setup.sh "${COMPILERS}" \
+    && bash /tmp/library-scripts/compiler-download.sh "/" "${COMPILERS}" \
+    && bash /tmp/library-scripts/compiler-setup.sh \
     && apt-get install --no-install-recommends -y clang-format python3-defusedxml python3-distutils \
     && apt-get clean -y && rm -rf /var/lib/apt/lists/* /tmp/library-scripts

.devcontainer/library-scripts/compiler-download.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+export XZ_OPT='-T0'
+
+install_dir="$1"
+shift
+compilers_list="$1"
+shift
+bambuhls_compiler_url="https://release.bambuhls.eu/compiler"
+
+inflate() {
+   echo "Installing $1 into $2"
+   case $1 in
+      clang-12 )
+         wget https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
+         ;;
+      clang-11 )
+         wget https://github.com/llvm/llvm-project/releases/download/llvmorg-11.1.0/clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
+         ;;
+      clang-10 )
+         wget https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.1/clang+llvm-10.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
+         ;;
+      clang-9 )
+         wget https://github.com/llvm/llvm-project/releases/download/llvmorg-9.0.1/clang+llvm-9.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
+         ;;
+      clang-8 )
+         wget https://releases.llvm.org/8.0.0/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
+         ;;
+      clang-7 )
+         wget https://releases.llvm.org/7.0.1/clang+llvm-7.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
+         ;;
+      clang-6 )
+         wget https://releases.llvm.org/6.0.1/clang+llvm-6.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
+         ;;
+      clang-5 )
+         wget https://releases.llvm.org/5.0.2/clang+llvm-5.0.2-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
+         ;;
+      clang-4 )
+         wget https://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
+         ;;
+      gcc-4.5 )
+         wget ${bambuhls_compiler_url}/gcc-4.5-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
+         ;;
+      gcc-4.6 )
+         wget ${bambuhls_compiler_url}/gcc-4.6-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
+         ;;
+      gcc-4.7 )
+         wget ${bambuhls_compiler_url}/gcc-4.7-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
+         ;;
+      gcc-4.8 )
+         wget ${bambuhls_compiler_url}/gcc-4.8-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
+         ;;
+      gcc-4.9 )
+         wget ${bambuhls_compiler_url}/gcc-4.9-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
+         ;;
+      gcc-5 )
+         wget ${bambuhls_compiler_url}/gcc-5-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
+         ;;
+      gcc-6 )
+         wget ${bambuhls_compiler_url}/gcc-6-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
+         ;;
+      gcc-7 )
+         wget ${bambuhls_compiler_url}/gcc-7-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
+         ;;
+      gcc-8 )
+         wget ${bambuhls_compiler_url}/gcc-8-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
+         ;;
+      * )
+         ;;
+   esac
+} 
+IFS=',' read -r -a compilers <<< "${compilers_list}"
+compilers=( $(IFS=$'\n'; echo "${compilers[*]}" | sort -V) )
+for compiler in "${compilers[@]}"
+do
+inflate $compiler $install_dir
+done
+wait

.devcontainer/library-scripts/compiler-setup.sh

@@ -8,78 +8,6 @@
 #     versions of GCC compilers after newer ones may cause compatibility issues.
 ###
 set -e
-export XZ_OPT='-T0'
-
-install="$1"
-
-inflate() {
-   echo "Installing $1"
-   case $1 in
-      clang-12 )
-         wget https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
-         ;;
-      clang-11 )
-         wget https://github.com/llvm/llvm-project/releases/download/llvmorg-11.1.0/clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
-         ;;
-      clang-10 )
-         wget https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.1/clang+llvm-10.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
-         ;;
-      clang-9 )
-         wget https://github.com/llvm/llvm-project/releases/download/llvmorg-9.0.1/clang+llvm-9.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
-         ;;
-      clang-8 )
-         wget https://releases.llvm.org/8.0.0/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
-         ;;
-      clang-7 )
-         wget https://releases.llvm.org/7.0.1/clang+llvm-7.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
-         ;;
-      clang-6 )
-         wget https://releases.llvm.org/6.0.1/clang+llvm-6.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
-         ;;
-      clang-5 )
-         wget https://releases.llvm.org/5.0.2/clang+llvm-5.0.2-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
-         ;;
-      clang-4 )
-         wget https://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz -nv -O - | tar -C $2 -xJf - &
-         ;;
-      gcc-4.5 )
-         wget https://release.bambuhls.eu/compiler/gcc-4.5-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
-         ;;
-      gcc-4.6 )
-         wget https://release.bambuhls.eu/compiler/gcc-4.6-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
-         ;;
-      gcc-4.7 )
-         wget https://release.bambuhls.eu/compiler/gcc-4.7-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
-         ;;
-      gcc-4.8 )
-         wget https://release.bambuhls.eu/compiler/gcc-4.8-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
-         ;;
-      gcc-4.9 )
-         wget https://release.bambuhls.eu/compiler/gcc-4.9-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
-         ;;
-      gcc-5 )
-         wget https://release.bambuhls.eu/compiler/gcc-5-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
-         ;;
-      gcc-6 )
-         wget https://release.bambuhls.eu/compiler/gcc-6-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
-         ;;
-      gcc-7 )
-         wget https://release.bambuhls.eu/compiler/gcc-7-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
-         ;;
-      gcc-8 )
-         wget https://release.bambuhls.eu/compiler/gcc-8-bambu-Ubuntu_16.04.tar.xz --no-check-certificate -nv -O - | tar -C $2 -xJf - 
-         ;;
-      * )
-         ;;
-   esac
-} 
-IFS=',' read -r -a compilers <<< "${install}"
-compilers=( $(IFS=$'\n'; echo "${compilers[*]}" | sort -V) )
-for compiler in "${compilers[@]}"
-do
-inflate $compiler /
-done
-wait
 
 if [[ "$PATH" != */usr/lib/ccache* ]]; then
    echo "export PATH=/usr/lib/ccache:$PATH" >> /etc/profile

.github/SETUP.md

@@ -1,10 +1,9 @@
 # Expected github runners setup
 
-## Job configuration
-A worflow job may be configured to run within a docker machine or directly in the host environment, in the latter case be aware that Bambu distributions coming from other jobs as artifacts may need the _APPDIR_ variable to be set to the new install location.
+Github runners may be configured to run within a docker container or directly in the host environment. In the former case there may be some issues during the execution of external vendor tools which may not support a containerized environment.
 
 ## Runner labels
-Runners may expose many different labels based on what tools are available on the host machine.
+Runners may expose many different labels based on what tools are available on the host machine. Comments about docker volumes are intender for a containerized environment only.
 
 - **altera**: Altera synthesis tools are available on the host machine and a docker volume named altera-tools exposes Altera tools install directories (e.g. Quartus, QuestaSim, ...) 
 - **intel**: Intel synthesis tools are available on the host machine and a docker volume named intel-tools exposes modern Intel FPGA tools install directories (e.g. Quartus Prime, QuestaSim, ...)
@@ -13,20 +12,28 @@ Runners may expose many different labels based on what tools are available on th
 - **nanoxplore**: NanoXplore synthesis tools are available on the host machine and a docker volume named nanoxplore-tools exposes NanoXplore tools install directories
 - **xilinx**: Xilinx synthesis tools are available on the host machine and a docker volume named xilinx-tools exposes Xilinx tools install directories (e.g. Vivado, Vitis HLS, ...)
 
-Directories containing license files should be copied or linked in container user home. Volumes may be defined as read-only to avoid issues.
+If working with docker containers, directories containing license files should be copied or linked in container user home. Volumes may be defined as read-only to avoid issues.
 
-A **licenses-home** volume is always expected when at least one of the above is defined. It should contain all necessary license files for available tools. Workflow job is expected to copy the volume content into the home directory of current user when running a container.
-
-To create such volumes the following may be useful:
-
-```
-docker volume create --driver local --opt o=bind,ro --opt type=none --opt device=/path/to/dir vendor-tools
-```
-
-## Runner environment
+## Environment variables
 Some environment variables are expected to be set by each runner host:
 
 - **J**: number of maximum parallel jobs handled by the runner
 - **LM_LICENSE_FILE**: license file path for simulation/synthesis tools
+- **NXLMD_LICENSE_FILE**: NanoXplore license file path (needed only if different from LM_LICENSE_FILE)
 - **NANOXPLORE_BYPASS**: NanoXplore bypass setting
 - **LIBRARY_PATH**: necessary to support older gcc compilers (set to: /usr/lib/x86_64-linux-gnu)
+
+## Python support
+Current CI implementation requires Python 3.6.15 to be available in the runner environment. Pyenv is recommended to provide the support.
+Furthermore pip packages from `etc/scripts/requirements.txt` are required to run python scripts correctly.
+
+Use the following to install the required Python version through PyEnv and set it as global default.
+
+```
+CONFIGURE_OPTS="--enable-shared" pyenv install 3.6.15
+pyenv global 3.6.15
+pip install -r /path/to/repo/et/scripts/requirements.txt
+```
+
+Note that Github Runners are launched as systemd services, thus `~/.bashrc` or `~/.profile` are not loaded.
+A `.path` file should be added in the runner directory containing standard _PATH_ variable prepended with PyEnv shims and bin paths.

.github/actions/generate-appimage/action.yml

@@ -0,0 +1,23 @@
+name: "Panda build"
+description: "Configure, build, and generate Panda Bambu appimage"
+inputs:
+  config-args:
+    description: "Configure command line arguments"
+    required: false
+    default: ""
+  jobs:
+    description: "Parallel jobs"
+    required: false
+    default: "1"
+outputs:
+  appimage:
+    description: "Output appimage file"
+  dist-dir:
+    description: "Distribution directory with appimage contents"
+runs:
+  using: "docker"
+  image: "Dockerfile"
+  env:
+    J: ${{inputs.jobs}}
+  args:
+    - ${{inputs.config-args}}

.github/actions/generate-bin/entrypoint.sh → .github/actions/generate-appimage/entrypoint.sh

@@ -19,7 +19,11 @@
 ###
 set -e
 
-workspace_dir=$PWD
+workspace_dir="$PWD"
+dist_dir="$workspace_dir/dist"
+build_dir="$workspace_dir/build"
+ccache_dir="$workspace_dir/.ccache"
+autoconf_cache_dir="$workspace_dir/.autoconf"
 
 function cleanup {
    echo "::endgroup::"
@@ -36,11 +40,11 @@ export PATH=/usr/lib/ccache:$PATH
 mkdir -p ~/.ccache/
 cat > ~/.ccache/ccache.conf << EOF
 max_size = 5.0G
-cache_dir = $workspace_dir/.ccache
+cache_dir = $ccache_dir
 EOF
-if [[ -d "dist" ]]; then
+if [[ -d "$dist_dir" ]]; then
    echo "Pre-initialized dist dir found. Installing system wide..."
-   cp -r dist/. /
+   cp -r $dist_dir/. /
 fi
 
 if [[ -d "compiler" ]]; then
@@ -57,12 +61,12 @@ do
    NO_DELETE+=" -o -name $(basename $bin)"
 done
 NO_DELETE+=" ${CLANG_EXES[@]/#/-o -name }"
-find dist/clang+llvm-*/bin '(' -type f -o -type l ')' ! '(' $NO_DELETE ')' -delete
-rm -f dist/clang+llvm*/lib/*.a
-rm -rf dist/clang+llvm*/share
-rm -rf dist/usr/share
+find $dist_dir/clang+llvm-*/bin '(' -type f -o -type l ')' ! '(' $NO_DELETE ')' -delete
+rm -f $dist_dir/clang+llvm*/lib/*.a
+rm -rf $dist_dir/clang+llvm*/share
+rm -rf $dist_dir/usr/share
 
-mkdir -p "$workspace_dir/dist/usr/bin"
+mkdir -p "$dist_dir/usr/bin"
 for clang_exe in $CLANG_BINS
 do
    CLANG_VER=$(sed 's/clang-//g' <<< "$(basename $clang_exe)")
@@ -72,7 +76,7 @@ do
    do
       if [[ -f "$CLANG_DIR/$app" ]]; then
          ln -sf "$CLANG_DIR/$app" "/usr/bin/$app-$CLANG_VER"
-         ln -sf "../..$CLANG_DIR/$app" "$workspace_dir/dist/usr/bin/$app-$CLANG_VER"
+         ln -sf "../..$CLANG_DIR/$app" "$dist_dir/usr/bin/$app-$CLANG_VER"
       fi
    done
    echo "Generating ccache alias for clang-$CLANG_VER"
@@ -87,7 +91,7 @@ do
    ln -sf ../../bin/ccache "/usr/lib/ccache/$(basename $compiler)"
 done
 
-max_gcc_ver="$(ls -x -v -1a dist/usr/include/c++ 2> /dev/null | tail -1)"
+max_gcc_ver="$(ls -x -v -1a $dist_dir/usr/include/c++ 2> /dev/null | tail -1)"
 if [[ -z "${max_gcc_ver}" ]]
 then
   echo "At least one gcc version must be bundled in the AppImage"
@@ -100,28 +104,39 @@ make -f Makefile.init
 echo "::endgroup::"
 
 echo "::group::Configure build environment"
-mkdir build
-cd build
-../configure --prefix=/usr $@
-cd ..
+mkdir -p $build_dir
+cd $build_dir
+if [[ -d "$autoconf_cache_dir" ]]; then
+   echo "Restoring autoconf cache"
+   mv $autoconf_cache_dir/* .
+fi
+../configure --prefix=/usr -C $@
+autoconf_caches=("`find . -type f -path '**/config.cache'`")
+for cache in $autoconf_caches
+do
+   mirror_dir="$autoconf_cache_dir/$(dirname $cache)"
+   mkdir -p $mirror_dir
+   cp $cache $mirror_dir
+done
+cd $workspace_dir
 echo "::endgroup::"
 
 echo "::group::Build Bambu"
-make --directory=build -j$J install-strip DESTDIR="$workspace_dir/dist"
+make --directory=$build_dir -j$J install-strip DESTDIR="$dist_dir"
 echo "::endgroup"
 
 echo "::group::Package Appimage"
 
 echo "Inflating libraries..."
-mkdir dist/lib
-mkdir dist/lib/x86_64-linux-gnu/
-cp -d /lib/x86_64-linux-gnu/libtinfo.so* dist/lib/x86_64-linux-gnu/
-cp -d /usr/lib/x86_64-linux-gnu/libicu*.so* dist/lib/x86_64-linux-gnu/
-cp -d /usr/lib/libbdd.so* dist/usr/lib/
+mkdir $dist_dir/lib
+mkdir $dist_dir/lib/x86_64-linux-gnu/
+cp -d /lib/x86_64-linux-gnu/libtinfo.so* $dist_dir/lib/x86_64-linux-gnu/
+cp -d /usr/lib/x86_64-linux-gnu/libicu*.so* $dist_dir/lib/x86_64-linux-gnu/
+cp -d /usr/lib/libbdd.so* $dist_dir/usr/lib/
 
 echo "Inflating metadata..."
-cp style/img/panda.png.in dist/bambu.png
-cat > dist/bambu.desktop << EOF
+cp style/img/panda.png.in $dist_dir/bambu.png
+cat > $dist_dir/bambu.desktop << EOF
 [Desktop Entry]
 Name=bambu
 Exec=tool_select.sh
@@ -130,9 +145,11 @@ Type=Application
 Terminal=true
 Categories=Development;
 EOF
-cat > dist/usr/bin/tool_select.sh << EOF
+cat > $dist_dir/usr/bin/tool_select.sh << EOF
 #!/bin/bash
 export LC_ALL="C"
+unset PYTHONHOME  # Python is not bundled with this AppImage
+unset PYTHONPATH
 BINARY_NAME=\$(basename "\$ARGV0")
 BINARY_PATH="\$APPDIR/usr/bin/\$BINARY_NAME"
 if [ "\$BINARY_NAME" == "debug_terminal" ]; then
@@ -143,12 +160,12 @@ if [ ! -e "\$BINARY_PATH" ]; then
 fi
 \$BINARY_PATH "\$@"
 EOF
-chmod a+x dist/usr/bin/tool_select.sh
+chmod a+x $dist_dir/usr/bin/tool_select.sh
 
 echo "Generating appimage..."
-curl -L https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-x86_64 -o dist/AppRun -s
-chmod +x dist/AppRun
-ARCH=x86_64 appimagetool dist 2> /dev/null
+curl -L https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-x86_64 -o $dist_dir/AppRun -s
+chmod +x $dist_dir/AppRun
+ARCH=x86_64 appimagetool $dist_dir 2> /dev/null
 
 echo "::set-output name=appimage::$(ls *.AppImage)"
 echo "::set-output name=dist-dir::dist"