From 409a72962113ca1655636a627c8ccf1c8ba9af11 Mon Sep 17 00:00:00 2001 From: one Date: Thu, 4 Apr 2024 11:03:10 +0800 Subject: [PATCH 01/73] Update yum sources --- etc/apt/sources.list | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/etc/apt/sources.list b/etc/apt/sources.list index 50137fc..a247bbf 100644 --- a/etc/apt/sources.list +++ b/etc/apt/sources.list @@ -1,13 +1,13 @@ # 默认注释了源码镜像以提高 apt update 速度，如有需要可自行取消注释 -deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic main restricted universe multiverse -# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic main restricted universe multiverse -deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-updates main restricted universe multiverse -# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-updates main restricted universe multiverse -deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-backports main restricted universe multiverse -# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-backports main restricted universe multiverse -deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-security main restricted universe multiverse -# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-security main restricted universe multiverse +deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal main restricted universe multiverse +# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal main restricted universe multiverse +deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-updates main restricted universe multiverse +# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-updates main restricted universe multiverse +deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-backports main restricted universe multiverse +# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-backports main restricted universe multiverse +deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-security main restricted universe multiverse +# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-security main restricted universe multiverse # 预发布软件源，不建议启用 -# deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-proposed main restricted universe multiverse -# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-proposed main restricted universe multiverse +# deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse +# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse From 3739706b17a16fdc198d15c5e3e3c86973708bec Mon Sep 17 00:00:00 2001 From: one Date: Thu, 4 Apr 2024 15:08:54 +0800 Subject: [PATCH 02/73] Remove rocm and spack mirror create commands for now --- Dockerfile | 102 +++++++++++++++++++---------------------------------- Makefile | 10 +++--- 2 files changed, 43 insertions(+), 69 deletions(-) diff --git a/Dockerfile b/Dockerfile index b981d9c..4ae52a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,11 @@ #=============================================================================== # Stage 1: build packages #=============================================================================== -ARG SPACK_IMAGE="spack/ubuntu-bionic" -ARG SPACK_VERSION="v0.17.1" +ARG SPACK_VERSION="0.21.2" +ARG SPACK_IMAGE="spack/ubuntu-focal" FROM ${SPACK_IMAGE}:${SPACK_VERSION} AS builder +ARG UBUNTU_CODE +ENV UBUNTU_CODE=${UBUNTU_CODE:-"focal"} #------------------------------------------------------------------------------- # Set up environments @@ -12,22 +14,27 @@ USER root WORKDIR /tmp # set Spack paths which should be shared between docker stages -ARG SPACK_ROOT=/opt/spack -ARG CONFIG_DIR=/etc/spack -ARG INSTALL_DIR=/opt/software -ARG MIRROR_DIR=/opt/mirror -ARG REPO_DIR=/opt/repo +ENV SPACK_ROOT=/opt/spack +ENV CONFIG_DIR=/etc/spack +ENV INSTALL_DIR=/opt/software +ENV MIRROR_DIR=/opt/mirror +ENV REPO_DIR=/opt/repo # create directories for Spack RUN set -e; \ mkdir -p $CONFIG_DIR; \ mkdir -p $INSTALL_DIR; \ + mkdir -p $REPO_DIR; \ mkdir -p $MIRROR_DIR # copy files from the context to the image COPY etc/apt/ /etc/apt/ -#COPY mirror/ $MIRROR_DIR/ COPY repo/ $REPO_DIR/ +# to hold a local package mirror as needed +#COPY mirror/ $MIRROR_DIR/ + +# Update source list +RUN sed -i -e "s/focal/$UBUNTU_CODE/g" /etc/apt/sources.list # set the arch for packages ARG TARGET="x86_64" @@ -36,7 +43,7 @@ ARG TARGET="x86_64" RUN (echo "config:" \ && echo " install_tree:" \ && echo " root: $INSTALL_DIR" \ -&& echo " connect_timeout: 0") > $CONFIG_DIR/config.yaml +&& echo " connect_timeout: 600") > $CONFIG_DIR/config.yaml RUN (echo "mirrors:" \ && echo " local: file://$MIRROR_DIR") > $CONFIG_DIR/mirrors.yaml @@ -51,23 +58,11 @@ RUN (echo "packages:" \ #------------------------------------------------------------------------------- # Find or install compilers #------------------------------------------------------------------------------- -# install apt repositries -RUN apt-get update && apt-get install -y \ - apt-transport-https \ - ca-certificates \ - gnupg \ - software-properties-common \ - wget \ -&& wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \ - gpg --dearmor - | \ - tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null \ -&& apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' - # install LLVM and CMake RUN apt-get update && apt-get install -y \ - llvm-9 \ - clang-9 \ - libomp-9-dev \ + llvm-10 \ + clang-10 \ + libomp-10-dev \ cmake # find external packages @@ -77,13 +72,14 @@ RUN spack external find --scope system --not-buildable \ autoconf \ automake \ cmake \ + gmake \ libtool \ perl # find system gcc and clang RUN spack compiler find; \ spack config get compilers > $CONFIG_DIR/compilers.yaml; \ - spack compilers + spack compiler list #------------------------------------------------------------------------------- # Install dependencies for antmoc @@ -96,22 +92,18 @@ ARG CLANG_SPEC="clang" ARG MPICH_SPEC="mpich~fortran" ARG OPENMPI_SPEC="openmpi" -RUN spack mirror create -D -d $MIRROR_DIR cmake %$GCC_SPEC \ -&& spack install --fail-fast -ny --reuse cmake %$GCC_SPEC -RUN spack mirror create -D -d $MIRROR_DIR lcov@1.14 %$GCC_SPEC \ -&& spack install --fail-fast -ny --reuse lcov@1.14 %$GCC_SPEC -RUN spack mirror create -D -d $MIRROR_DIR antmoc %$CLANG_SPEC ~mpi \ -&& spack install --fail-fast -ny --reuse antmoc %$CLANG_SPEC ~mpi -RUN spack mirror create -D -d $MIRROR_DIR antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC \ -&& spack install --fail-fast -ny --reuse antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC -RUN spack mirror create -D -d $MIRROR_DIR antmoc %$GCC_SPEC ~mpi \ -&& spack install --fail-fast -ny --reuse antmoc %$GCC_SPEC ~mpi -RUN spack mirror create -D -d $MIRROR_DIR antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC \ -&& spack install --fail-fast -ny --reuse antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC -RUN spack mirror create -D -d $MIRROR_DIR antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC \ -&& spack install --fail-fast -ny --reuse antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC +RUN spack install --fail-fast -ny cmake %$GCC_SPEC +RUN spack install --fail-fast -ny lcov@=2.0 %$GCC_SPEC +RUN spack install --fail-fast -ny antmoc %$CLANG_SPEC ~mpi +RUN spack install --fail-fast -ny antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC +RUN spack install --fail-fast -ny antmoc %$GCC_SPEC ~mpi +RUN spack install --fail-fast -ny antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC +RUN spack install --fail-fast -ny antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC RUN spack gc -y && spack clean -a +# Check spack and dependency installation +RUN spack debug report && spack find -v + #=============================================================================== # Stage 2: build the runtime environment @@ -125,10 +117,6 @@ LABEL maintainer="An Wang " #------------------------------------------------------------------------------- # Copy artifacts from stage 1 to stage 2 #------------------------------------------------------------------------------- -ARG CONFIG_DIR=/etc/spack -ARG INSTALL_DIR=/opt/software -ARG REPO_DIR=/opt/repo - COPY --from=builder $CONFIG_DIR $CONFIG_DIR COPY --from=builder $INSTALL_DIR $INSTALL_DIR COPY --from=builder $REPO_DIR $REPO_DIR @@ -138,36 +126,20 @@ COPY --from=builder $REPO_DIR $REPO_DIR #------------------------------------------------------------------------------- # install apt repositries COPY etc/apt/ /etc/apt/ -RUN apt-get update && apt-get install -y \ - apt-transport-https \ - ca-certificates \ - gnupg \ - software-properties-common \ - wget \ -&& wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \ - gpg --dearmor - | \ - tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null \ -&& apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' \ - \ -&& wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | \ - apt-key add - \ -&& echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ xenial main' | \ - tee /etc/apt/sources.list.d/rocm.list # install CMake, LLVM, and ROCm RUN apt-get update && apt-get install -y --no-install-recommends \ + wget \ cmake \ - clang-9 \ - llvm-9 \ - libomp-9-dev \ + clang-10 \ + llvm-10 \ + libomp-10-dev \ kmod \ openssh-server \ sudo \ - rocm-dev \ - rocthrust \ && rm -f /usr/bin/clang /usr/bin/clang++ \ -&& ln -s /usr/bin/clang-9 /usr/bin/clang \ -&& ln -s /usr/bin/clang++-9 /usr/bin/clang++ \ +&& ln -s /usr/bin/clang-10 /usr/bin/clang \ +&& ln -s /usr/bin/clang++-10 /usr/bin/clang++ \ && rm -rf /var/lib/apt/lists/* #------------------------------------------------------------------------------- diff --git a/Makefile b/Makefile index 4fcc3a1..72669eb 100644 --- a/Makefile +++ b/Makefile @@ -3,15 +3,16 @@ #=============================================================================== # Build-time arguments -SPACK_IMAGE ?= "spack/ubuntu-bionic" -SPACK_VERSION ?= "latest" +UBUNTU_CODE ?= "focal" +SPACK_VERSION ?= "0.21.2" +SPACK_IMAGE = "spack/ubuntu-$(UBUNTU_CODE)" # Target TARGET ?= "x86_64" # Image name DOCKER_IMAGE ?= antmoc/antmoc-ci -DOCKER_TAG := 0.1.15 +DOCKER_TAG := 0.1.16-a # Default user USER_NAME ?= hpcer @@ -39,8 +40,9 @@ release: docker_build docker_push output docker_build: # Build Docker image docker build \ - --build-arg SPACK_IMAGE=$(SPACK_IMAGE) \ + --build-arg UBUNTU_CODE=$(UBUNTU_CODE) \ --build-arg SPACK_VERSION=$(SPACK_VERSION) \ + --build-arg SPACK_IMAGE=$(SPACK_IMAGE) \ --build-arg TARGET=$(TARGET) \ --build-arg BUILD_DATE=$(BUILD_DATE) \ --build-arg VCS_URL=$(VCS_URL) \ From 62709027f5c01007bf90ba8d679486de8fbb4226 Mon Sep 17 00:00:00 2001 From: one Date: Thu, 4 Apr 2024 15:10:48 +0800 Subject: [PATCH 03/73] Update spack package.py --- .gitignore | 1 + repo/packages/antmoc/package.py | 2 +- repo/packages/cxxopts/package.py | 63 +++++++++++++++---------------- repo/packages/tinyxml2/package.py | 27 +++++++++++++ repo/packages/toml11/package.py | 29 ++++++++++++++ 5 files changed, 89 insertions(+), 33 deletions(-) create mode 100644 .gitignore create mode 100644 repo/packages/tinyxml2/package.py create mode 100644 repo/packages/toml11/package.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/repo/packages/antmoc/package.py b/repo/packages/antmoc/package.py index 0ee35c3..83032b2 100644 --- a/repo/packages/antmoc/package.py +++ b/repo/packages/antmoc/package.py @@ -14,7 +14,7 @@ class Antmoc(BundlePackage): depends_on('cmake@3.16:', type='build') depends_on('mpi@3.0:', when='+mpi', type=('build', 'link', 'run')) - depends_on('cxxopts@master') + depends_on('cxxopts@3.0.0:') depends_on('fmt@6.0.0:') depends_on('tinyxml2@7.0.0:') depends_on('toml11@3.6.0:') diff --git a/repo/packages/cxxopts/package.py b/repo/packages/cxxopts/package.py index 3acaaac..be93b96 100644 --- a/repo/packages/cxxopts/package.py +++ b/repo/packages/cxxopts/package.py @@ -1,44 +1,43 @@ -# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack import * +from spack.package import * class Cxxopts(CMakePackage): """Lightweight C++ command line option parser""" homepage = "https://github.com/jarro2783/cxxopts" - url = "https://github.com/jarro2783/cxxopts/archive/v2.2.0.tar.gz" - git = "https://github.com/jarro2783/cxxopts.git" - - version('master', branch='master') - version('2.2.0', sha256='447dbfc2361fce9742c5d1c9cfb25731c977b405f9085a738fbd608626da8a4d') - version('2.1.2', sha256='95f524c6615a2067e935e02ef74b013b17efa339df0a3c9db3e91fc0afbaf269') - version('2.1.1', sha256='e19216251427d04f0273e6487c0246ae2dbb4154bf178f43896af8fa1ef89f3f') - version('2.1.0', sha256='7672c79e5e48dd0cb1935c6efe65d2695d96fa764bf06c493d2db25a3cf633b4') - version('2.0.0', sha256='f575a12645743032c27a7bc661e8598f249a8453d7f5388bcae66ac3b089399e') - version('1.4.4', sha256='1d0eedb39ecbc64a0f82d8b6fe40d5c8e611501702969cfbd14a07ce6ddb8501') - version('1.4.3', sha256='9103d6d75a3f599728256ce89922a8cd348bfe5874de62ca1436f605f194c52e') - version('1.4.2', sha256='8fc5e16e68ddf549465f8bec6b56adaccfab9d183093616ddd4d3e80115844cb') - version('1.4.1', sha256='c5ccfe99bd3db0604d077c968b39a42d61333a64d171fe84d7037d6c0dcc996d') - version('1.4.0', sha256='60d4a482ec603ef4efa2603978596716884b33e24d39af6ddca52b4a30f7107b') - - variant('unicode', default=False, - description='Enables unicode support using the ICU library.') - - depends_on('cmake@3.1.0:', type='build') - depends_on('icu4c', when='+unicode') + url = "https://github.com/jarro2783/cxxopts/archive/v2.2.0.tar.gz" + maintainers("haampie") + + license("MIT") + + version("3.1.1", sha256="523175f792eb0ff04f9e653c90746c12655f10cb70f1d5e6d6d9491420298a08") + version("3.0.0", sha256="36f41fa2a46b3c1466613b63f3fa73dc24d912bc90d667147f1e43215a8c6d00") + version("2.2.1", sha256="984aa3c8917d649b14d7f6277104ce38dd142ce378a9198ec926f03302399681") + version("2.2.0", sha256="447dbfc2361fce9742c5d1c9cfb25731c977b405f9085a738fbd608626da8a4d") + version("2.1.2", sha256="95f524c6615a2067e935e02ef74b013b17efa339df0a3c9db3e91fc0afbaf269") + version("2.1.1", sha256="e19216251427d04f0273e6487c0246ae2dbb4154bf178f43896af8fa1ef89f3f") + version("2.1.0", sha256="7672c79e5e48dd0cb1935c6efe65d2695d96fa764bf06c493d2db25a3cf633b4") + version("2.0.0", sha256="f575a12645743032c27a7bc661e8598f249a8453d7f5388bcae66ac3b089399e") + version("1.4.4", sha256="1d0eedb39ecbc64a0f82d8b6fe40d5c8e611501702969cfbd14a07ce6ddb8501") + version("1.4.3", sha256="9103d6d75a3f599728256ce89922a8cd348bfe5874de62ca1436f605f194c52e") + version("1.4.2", sha256="8fc5e16e68ddf549465f8bec6b56adaccfab9d183093616ddd4d3e80115844cb") + version("1.4.1", sha256="c5ccfe99bd3db0604d077c968b39a42d61333a64d171fe84d7037d6c0dcc996d") + version("1.4.0", sha256="60d4a482ec603ef4efa2603978596716884b33e24d39af6ddca52b4a30f7107b") + + variant("unicode", default=False, description="Enables unicode support using the ICU library.") + + depends_on("cmake@3.1.0:", type="build") + depends_on("icu4c", when="+unicode") def cmake_args(self): - args = ['-DCXXOPTS_ENABLE_INSTALL=ON', - '-DCXXOPTS_BUILD_EXAMPLES=OFF', - '-DCXXOPTS_BUILD_TESTS=OFF'] - - if '+unicode' in self.spec: - args.append('-DCXXOPTS_USE_UNICODE_HELP=ON') - else: - args.append('-DCXXOPTS_USE_UNICODE_HELP=OFF') - - return args + return [ + self.define("CXXOPTS_ENABLE_INSTALL", "ON"), + self.define("CXXOPTS_BUILD_EXAMPLES", "OFF"), + self.define("CXXOPTS_BUILD_TESTS", "OFF"), + self.define_from_variant("CXXOPTS_USE_UNICODE_HELP", "unicode"), + ] diff --git a/repo/packages/tinyxml2/package.py b/repo/packages/tinyxml2/package.py new file mode 100644 index 0000000..c4871ef --- /dev/null +++ b/repo/packages/tinyxml2/package.py @@ -0,0 +1,27 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * + + +class Tinyxml2(CMakePackage): + """Simple, small, efficient, C++ XML parser""" + + homepage = "http://grinninglizard.com/tinyxml2/" + url = "https://github.com/leethomason/tinyxml2/archive/3.0.0.tar.gz" + + license("Zlib") + + version("10.0.0", sha256="3bdf15128ba16686e69bce256cc468e76c7b94ff2c7f391cc5ec09e40bff3839") + version("9.0.0", sha256="cc2f1417c308b1f6acc54f88eb70771a0bf65f76282ce5c40e54cfe52952702c") + version("8.0.0", sha256="6ce574fbb46751842d23089485ae73d3db12c1b6639cda7721bf3a7ee862012c") + version("7.0.0", sha256="fa0d1c745d65d4d833e62cb183e23c2034dc7a35ec1a4977e808bdebb9b4fe60") + version("6.2.0", sha256="cdf0c2179ae7a7931dba52463741cf59024198bbf9673bf08415bcb46344110f") + version("4.0.1", sha256="14b38ef25cc136d71339ceeafb4856bb638d486614103453eccd323849267f20") + version("4.0.0", sha256="90add44f06de081047d431c08d7269c25b4030e5fe19c3bc8381c001ce8f258c") + version("3.0.0", sha256="128aa1553e88403833e0cccf1b651f45ce87bc207871f53fdcc8e7f9ec795747") + version("2.2.0", sha256="f891224f32e7a06bf279290619cec80cc8ddc335c13696872195ffb87f5bce67") + version("2.1.0", sha256="4bdd6569fdce00460bf9cda0ff5dcff46d342b4595900d849cc46a277a74cce6") + version("2.0.2", sha256="3cc3aa09cd1ce77736f23488c7cb24e65e11daed4e870ddc8d352aa4070c7c74") diff --git a/repo/packages/toml11/package.py b/repo/packages/toml11/package.py new file mode 100644 index 0000000..b9b50b1 --- /dev/null +++ b/repo/packages/toml11/package.py @@ -0,0 +1,29 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +from spack.package import * + + +class Toml11(CMakePackage): + """toml11 is a C++11 (or later) header-only toml parser/encoder depending + only on C++ standard library.""" + + homepage = "https://github.com/ToruNiina/toml11" + url = "https://github.com/ToruNiina/toml11/archive/refs/tags/v3.7.1.tar.gz" + + maintainers("ashermancinelli", "ToruNiina") + + version("3.7.1", sha256="afeaa9aa0416d4b6b2cd3897ca55d9317084103077b32a852247d8efd4cf6068") + version("3.7.0", sha256="a0b6bec77c0e418eea7d270a4437510884f2fe8f61e7ab121729624f04c4b58e") + version("3.6.1", sha256="ca4c390ed8da0d77ae6eca30e70ab0bf5cc92adfc1bc2f71a2066bc5656d8d96") + version("3.6.0", sha256="39e8d651db346ae8c7e3b39d6338a37232b9af3bba36ade45b241bf105c2226c") + version("3.5.0", sha256="fc613874c6e80dc740134a7353cf23c7f834b59cd601af84ab535ee16a53b1c3") + version("3.4.0", sha256="bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d") + version("3.3.1", sha256="0c1b29e1a2873a1b1f6d0865a2966a2d3fd9155aeccb6139cd5af22f70b0b08f") + version("3.3.0", sha256="b29995475922fae3095445219d36733ef18976abdc85685d0804ee3ea04f09c0") + version("3.2.1", sha256="370f17409cfcbf3f629728ed7ec2e1573544058615fb5d066f4f7c14693143a9") + version("3.2.0", sha256="3d54cac38ea24477190e0535377e824bf06562970ef4d35b59aa9729437e1019") + version("3.1.0", sha256="3a118f32e5343998f37be9807c72fd11c3168fe12a5b1abfdc0f1e60de6380a4") From a7c3a58e4951f79272a9f07b89cdde89530cfdb2 Mon Sep 17 00:00:00 2001 From: one Date: Thu, 4 Apr 2024 16:23:39 +0800 Subject: [PATCH 04/73] Update spack install commands --- Dockerfile | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4ae52a5..2695995 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,13 +27,14 @@ RUN set -e; \ mkdir -p $REPO_DIR; \ mkdir -p $MIRROR_DIR -# copy files from the context to the image -COPY etc/apt/ /etc/apt/ +# copy a self-hosted spack repo to the image COPY repo/ $REPO_DIR/ -# to hold a local package mirror as needed + +# hold a local package mirror as needed #COPY mirror/ $MIRROR_DIR/ -# Update source list +# update source list +COPY etc/apt/ /etc/apt/ RUN sed -i -e "s/focal/$UBUNTU_CODE/g" /etc/apt/sources.list # set the arch for packages @@ -92,13 +93,14 @@ ARG CLANG_SPEC="clang" ARG MPICH_SPEC="mpich~fortran" ARG OPENMPI_SPEC="openmpi" -RUN spack install --fail-fast -ny cmake %$GCC_SPEC -RUN spack install --fail-fast -ny lcov@=2.0 %$GCC_SPEC -RUN spack install --fail-fast -ny antmoc %$CLANG_SPEC ~mpi -RUN spack install --fail-fast -ny antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC -RUN spack install --fail-fast -ny antmoc %$GCC_SPEC ~mpi -RUN spack install --fail-fast -ny antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC -RUN spack install --fail-fast -ny antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC +RUN deps=("cmake %$GCC_SPEC" \ + "lcov@=2.0 %$GCC_SPEC" \ + "antmoc %$CLANG_SPEC ~mpi" \ + "antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC" \ + "antmoc %$GCC_SPEC ~mpi" \ + "antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC" \ + "antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC") \ + && for dep in "${deps[@]}"; do spack install --fail-fast -ny $dep; done RUN spack gc -y && spack clean -a # Check spack and dependency installation From 64340bb840ad765790ddbfff863b9851b244c2da Mon Sep 17 00:00:00 2001 From: one Date: Thu, 4 Apr 2024 17:13:30 +0800 Subject: [PATCH 05/73] Add rocm --- Dockerfile | 59 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2695995..e03c8c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,16 +60,38 @@ RUN (echo "packages:" \ # Find or install compilers #------------------------------------------------------------------------------- # install LLVM and CMake -RUN apt-get update && apt-get install -y \ - llvm-10 \ - clang-10 \ - libomp-10-dev \ - cmake +# RUN apt-get update && apt-get install -y \ +# llvm-10 \ +# clang-10 \ +# libomp-10-dev \ +# cmake + +# Register the ROCM package repository, and install rocm-dev package +ARG ROCM_VERSION=5.3 +ARG AMDGPU_VERSION=5.3 + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl libnuma-dev gnupg \ + && curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - \ + && printf "deb [arch=amd64] https://repo.radeon.com/rocm/apt/$ROCM_VERSION/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list \ + && printf "deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu focal main" | tee /etc/apt/sources.list.d/amdgpu.list \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + sudo \ + libelf1 \ + kmod \ + file \ + python3 \ + python3-pip \ + rocm-dev \ + build-essential && \ + apt-get clean # find external packages RUN spack external find --scope system --not-buildable \ gcc \ llvm \ + hip \ autoconf \ automake \ cmake \ @@ -81,6 +103,8 @@ RUN spack external find --scope system --not-buildable \ RUN spack compiler find; \ spack config get compilers > $CONFIG_DIR/compilers.yaml; \ spack compiler list +RUN spack external find --scope system --not-buildable --all +RUN spack debug report && spack find -v && exit 1 #------------------------------------------------------------------------------- # Install dependencies for antmoc @@ -93,14 +117,13 @@ ARG CLANG_SPEC="clang" ARG MPICH_SPEC="mpich~fortran" ARG OPENMPI_SPEC="openmpi" -RUN deps=("cmake %$GCC_SPEC" \ - "lcov@=2.0 %$GCC_SPEC" \ - "antmoc %$CLANG_SPEC ~mpi" \ - "antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC" \ - "antmoc %$GCC_SPEC ~mpi" \ - "antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC" \ - "antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC") \ - && for dep in "${deps[@]}"; do spack install --fail-fast -ny $dep; done +RUN spack install --fail-fast -ny cmake %$GCC_SPEC +RUN spack install --fail-fast -ny lcov@=2.0 %$GCC_SPEC +RUN spack install --fail-fast -ny antmoc %$CLANG_SPEC ~mpi +RUN spack install --fail-fast -ny antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC +RUN spack install --fail-fast -ny antmoc %$GCC_SPEC ~mpi +RUN spack install --fail-fast -ny antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC +RUN spack install --fail-fast -ny antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC RUN spack gc -y && spack clean -a # Check spack and dependency installation @@ -164,11 +187,11 @@ WORKDIR /home/$USER_NAME # generate a script for Spack RUN (echo "#!/usr/bin/env bash" \ -&& echo "export PATH=\$PATH:/opt/rocm/bin:/opt/rocm/rocprofiler/bin:/opt/rocm/opencl/bin" \ -&& echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/opt/rocm/lib:/opt/rocm/hip/lib:/opt/rocm/llvm/lib:/opt/rocm/opencl/lib" \ -&& echo "export INCLUDE=\$INCLUDE:/opt/rocm/include:/opt/rocm/hip/include:/opt/rocm/llvm/include" \ -&& echo "export C_INCLUDE_PATH=\$C_INCLUDE_PATH:/opt/rocm/include:/opt/rocm/hip/include:/opt/rocm/llvm/include" \ -&& echo "export CPLUS_INCLUDE_PATH=\$CPLUS_INCLUDE_PATH:/opt/rocm/include:/opt/rocm/hip/include:/opt/rocm/llvm/include" \ +# && echo "export PATH=\$PATH:/opt/rocm/bin:/opt/rocm/rocprofiler/bin:/opt/rocm/opencl/bin" \ +# && echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/opt/rocm/lib:/opt/rocm/hip/lib:/opt/rocm/llvm/lib:/opt/rocm/opencl/lib" \ +# && echo "export INCLUDE=\$INCLUDE:/opt/rocm/include:/opt/rocm/hip/include:/opt/rocm/llvm/include" \ +# && echo "export C_INCLUDE_PATH=\$C_INCLUDE_PATH:/opt/rocm/include:/opt/rocm/hip/include:/opt/rocm/llvm/include" \ +# && echo "export CPLUS_INCLUDE_PATH=\$CPLUS_INCLUDE_PATH:/opt/rocm/include:/opt/rocm/hip/include:/opt/rocm/llvm/include" \ && echo "export SPACK_ROOT=$SPACK_ROOT" \ && echo ". $SPACK_ROOT/share/spack/setup-env.sh" \ && echo "") > ~/setup-env.sh \ From 0a4b517167f344954dec5c825b8da70c4f65907f Mon Sep 17 00:00:00 2001 From: one Date: Thu, 4 Apr 2024 17:28:03 +0800 Subject: [PATCH 06/73] Remove stage 2 --- Dockerfile | 147 +++++++++++++++++------------------------------------ 1 file changed, 48 insertions(+), 99 deletions(-) diff --git a/Dockerfile b/Dockerfile index e03c8c8..9243c11 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,11 @@ -#=============================================================================== -# Stage 1: build packages -#=============================================================================== ARG SPACK_VERSION="0.21.2" ARG SPACK_IMAGE="spack/ubuntu-focal" FROM ${SPACK_IMAGE}:${SPACK_VERSION} AS builder ARG UBUNTU_CODE ENV UBUNTU_CODE=${UBUNTU_CODE:-"focal"} +LABEL maintainer="An Wang " + #------------------------------------------------------------------------------- # Set up environments #------------------------------------------------------------------------------- @@ -29,52 +28,42 @@ RUN set -e; \ # copy a self-hosted spack repo to the image COPY repo/ $REPO_DIR/ +COPY etc/apt/ /etc/apt/ # hold a local package mirror as needed #COPY mirror/ $MIRROR_DIR/ -# update source list -COPY etc/apt/ /etc/apt/ -RUN sed -i -e "s/focal/$UBUNTU_CODE/g" /etc/apt/sources.list - # set the arch for packages ARG TARGET="x86_64" # generate configurations RUN (echo "config:" \ -&& echo " install_tree:" \ -&& echo " root: $INSTALL_DIR" \ -&& echo " connect_timeout: 600") > $CONFIG_DIR/config.yaml - -RUN (echo "mirrors:" \ -&& echo " local: file://$MIRROR_DIR") > $CONFIG_DIR/mirrors.yaml - -RUN (echo "repos:" \ -&& echo " - $REPO_DIR") > $CONFIG_DIR/repos.yaml - -RUN (echo "packages:" \ -&& echo " all:" \ -&& echo " target: [$TARGET]") > $CONFIG_DIR/packages.yaml + && echo " install_tree:" \ + && echo " root: $INSTALL_DIR" \ + && echo " connect_timeout: 600") > $CONFIG_DIR/config.yaml \ + && (echo "mirrors:" \ + && echo " local: file://$MIRROR_DIR") > $CONFIG_DIR/mirrors.yaml \ + && (echo "repos:" \ + && echo " - $REPO_DIR") > $CONFIG_DIR/repos.yaml \ + && (echo "packages:" \ + && echo " all:" \ + && echo " target: [$TARGET]") > $CONFIG_DIR/packages.yaml #------------------------------------------------------------------------------- # Find or install compilers #------------------------------------------------------------------------------- -# install LLVM and CMake -# RUN apt-get update && apt-get install -y \ -# llvm-10 \ -# clang-10 \ -# libomp-10-dev \ -# cmake - # Register the ROCM package repository, and install rocm-dev package ARG ROCM_VERSION=5.3 ARG AMDGPU_VERSION=5.3 -RUN apt-get update \ +# install LLVM and CMake for spack, and +# install ROCm HIP, see https://github.com/ROCm/ROCm-docker/blob/master/dev/Dockerfile-ubuntu-20.04 +RUN sed -i -e "s/focal/$UBUNTU_CODE/g" /etc/apt/sources.list \ + && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl libnuma-dev gnupg \ && curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - \ && printf "deb [arch=amd64] https://repo.radeon.com/rocm/apt/$ROCM_VERSION/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list \ - && printf "deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu focal main" | tee /etc/apt/sources.list.d/amdgpu.list \ + && printf "deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu $UBUNTU_CODE main" | tee /etc/apt/sources.list.d/amdgpu.list \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ sudo \ @@ -84,27 +73,24 @@ RUN apt-get update \ python3 \ python3-pip \ rocm-dev \ - build-essential && \ - apt-get clean - -# find external packages -RUN spack external find --scope system --not-buildable \ - gcc \ - llvm \ - hip \ - autoconf \ - automake \ - cmake \ - gmake \ - libtool \ - perl - -# find system gcc and clang -RUN spack compiler find; \ - spack config get compilers > $CONFIG_DIR/compilers.yaml; \ - spack compiler list -RUN spack external find --scope system --not-buildable --all -RUN spack debug report && spack find -v && exit 1 + build-essential \ + llvm-10 clang-10 libomp-10-dev cmake openssh-server && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# find external packages and system compilers +RUN spack compiler find \ + && spack config get compilers > $CONFIG_DIR/compilers.yaml \ + && spack compiler list \ + && spack external find --scope system --not-buildable \ + gcc \ + llvm \ + autoconf \ + automake \ + cmake \ + gmake \ + libtool \ + perl #------------------------------------------------------------------------------- # Install dependencies for antmoc @@ -117,55 +103,18 @@ ARG CLANG_SPEC="clang" ARG MPICH_SPEC="mpich~fortran" ARG OPENMPI_SPEC="openmpi" -RUN spack install --fail-fast -ny cmake %$GCC_SPEC -RUN spack install --fail-fast -ny lcov@=2.0 %$GCC_SPEC -RUN spack install --fail-fast -ny antmoc %$CLANG_SPEC ~mpi -RUN spack install --fail-fast -ny antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC -RUN spack install --fail-fast -ny antmoc %$GCC_SPEC ~mpi -RUN spack install --fail-fast -ny antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC -RUN spack install --fail-fast -ny antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC -RUN spack gc -y && spack clean -a - -# Check spack and dependency installation -RUN spack debug report && spack find -v - - -#=============================================================================== -# Stage 2: build the runtime environment -#=============================================================================== -ARG SPACK_IMAGE -ARG SPACK_VERSION -FROM ${SPACK_IMAGE}:${SPACK_VERSION} - -LABEL maintainer="An Wang " - -#------------------------------------------------------------------------------- -# Copy artifacts from stage 1 to stage 2 -#------------------------------------------------------------------------------- -COPY --from=builder $CONFIG_DIR $CONFIG_DIR -COPY --from=builder $INSTALL_DIR $INSTALL_DIR -COPY --from=builder $REPO_DIR $REPO_DIR - -#------------------------------------------------------------------------------- -# Install system packages -#------------------------------------------------------------------------------- -# install apt repositries -COPY etc/apt/ /etc/apt/ +RUN deps=(\ + "cmake %$GCC_SPEC" \ + "lcov@=2.0 %$GCC_SPEC" \ + "antmoc %$CLANG_SPEC ~mpi" \ + "antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC" \ + "antmoc %$GCC_SPEC ~mpi" \ + "antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC" \ + "antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC") \ + && for dep in "${deps[@]}"; do spack install --fail-fast -ny $dep; done \ + && spack gc -y && spack clean -a \ + && spack debug report && spack find -v # Check spack and dependency installation -# install CMake, LLVM, and ROCm -RUN apt-get update && apt-get install -y --no-install-recommends \ - wget \ - cmake \ - clang-10 \ - llvm-10 \ - libomp-10-dev \ - kmod \ - openssh-server \ - sudo \ -&& rm -f /usr/bin/clang /usr/bin/clang++ \ -&& ln -s /usr/bin/clang-10 /usr/bin/clang \ -&& ln -s /usr/bin/clang++-10 /usr/bin/clang++ \ -&& rm -rf /var/lib/apt/lists/* #------------------------------------------------------------------------------- # Add a user @@ -176,7 +125,7 @@ ARG USER_NAME=hpcer # create the first user RUN set -e; \ \ - if ! id -u $USER_NAME > /dev/null 2>&1; then \ + if ! id -u $USER_NAME &> /dev/null; then \ useradd -m $USER_NAME; \ echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers; \ fi From 11a6176468eb8962d762dbe05194b67c34f44c10 Mon Sep 17 00:00:00 2001 From: one Date: Thu, 4 Apr 2024 23:01:54 +0800 Subject: [PATCH 07/73] Update dependency buinding order --- Dockerfile | 72 ++++++++++++++++--------------- Makefile | 2 +- repo/packages/antmoc/package.py | 19 ++++---- repo/packages/tinyxml2/package.py | 10 +++++ 4 files changed, 57 insertions(+), 46 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9243c11..5e37d69 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,12 +6,42 @@ ENV UBUNTU_CODE=${UBUNTU_CODE:-"focal"} LABEL maintainer="An Wang " -#------------------------------------------------------------------------------- -# Set up environments -#------------------------------------------------------------------------------- USER root WORKDIR /tmp +#------------------------------------------------------------------------------- +# Install system compilers and libraries +#------------------------------------------------------------------------------- +# Register the ROCM package repository, and install rocm-dev package +ARG ROCM_VERSION=5.3 +ARG AMDGPU_VERSION=5.3 + +# install LLVM and CMake for spack, and +# install ROCm HIP, see https://github.com/ROCm/ROCm-docker/blob/master/dev/Dockerfile-ubuntu-20.04 +COPY etc/apt/ /etc/apt/ +RUN sed -i -e "s/focal/$UBUNTU_CODE/g" /etc/apt/sources.list \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl libnuma-dev gnupg \ + && curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - \ + && printf "deb [arch=amd64] https://repo.radeon.com/rocm/apt/$ROCM_VERSION/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list \ + && printf "deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu $UBUNTU_CODE main" | tee /etc/apt/sources.list.d/amdgpu.list \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + sudo \ + libelf1 \ + kmod \ + file \ + python3 \ + python3-pip \ + rocm-dev \ + build-essential \ + llvm-10 clang-10 libomp-10-dev cmake openssh-server && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +#------------------------------------------------------------------------------- +# Set up spack +#------------------------------------------------------------------------------- # set Spack paths which should be shared between docker stages ENV SPACK_ROOT=/opt/spack ENV CONFIG_DIR=/etc/spack @@ -28,7 +58,6 @@ RUN set -e; \ # copy a self-hosted spack repo to the image COPY repo/ $REPO_DIR/ -COPY etc/apt/ /etc/apt/ # hold a local package mirror as needed #COPY mirror/ $MIRROR_DIR/ @@ -50,34 +79,8 @@ RUN (echo "config:" \ && echo " target: [$TARGET]") > $CONFIG_DIR/packages.yaml #------------------------------------------------------------------------------- -# Find or install compilers +# Find system compilers #------------------------------------------------------------------------------- -# Register the ROCM package repository, and install rocm-dev package -ARG ROCM_VERSION=5.3 -ARG AMDGPU_VERSION=5.3 - -# install LLVM and CMake for spack, and -# install ROCm HIP, see https://github.com/ROCm/ROCm-docker/blob/master/dev/Dockerfile-ubuntu-20.04 -RUN sed -i -e "s/focal/$UBUNTU_CODE/g" /etc/apt/sources.list \ - && apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl libnuma-dev gnupg \ - && curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - \ - && printf "deb [arch=amd64] https://repo.radeon.com/rocm/apt/$ROCM_VERSION/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list \ - && printf "deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu $UBUNTU_CODE main" | tee /etc/apt/sources.list.d/amdgpu.list \ - && apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - sudo \ - libelf1 \ - kmod \ - file \ - python3 \ - python3-pip \ - rocm-dev \ - build-essential \ - llvm-10 clang-10 libomp-10-dev cmake openssh-server && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - # find external packages and system compilers RUN spack compiler find \ && spack config get compilers > $CONFIG_DIR/compilers.yaml \ @@ -147,11 +150,10 @@ RUN (echo "#!/usr/bin/env bash" \ && chmod u+x ~/setup-env.sh #------------------------------------------------------------------------------- -# Reset the entrypoint +# Reset the entrypoint, add CMD #------------------------------------------------------------------------------- -ENTRYPOINT [] -CMD ["/bin/bash"] - +ENTRYPOINT ["/bin/bash"] +CMD ["interactive-shell"] #----------------------------------------------------------------------- # Build-time metadata as defined at http://label-schema.org diff --git a/Makefile b/Makefile index 72669eb..bba278e 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ TARGET ?= "x86_64" # Image name DOCKER_IMAGE ?= antmoc/antmoc-ci -DOCKER_TAG := 0.1.16-a +DOCKER_TAG := 0.1.16-alpha # Default user USER_NAME ?= hpcer diff --git a/repo/packages/antmoc/package.py b/repo/packages/antmoc/package.py index 83032b2..31d6b29 100644 --- a/repo/packages/antmoc/package.py +++ b/repo/packages/antmoc/package.py @@ -12,13 +12,12 @@ class Antmoc(BundlePackage): variant('mpi', default=False, description='Enable MPI support') - depends_on('cmake@3.16:', type='build') - depends_on('mpi@3.0:', when='+mpi', type=('build', 'link', 'run')) - depends_on('cxxopts@3.0.0:') - depends_on('fmt@6.0.0:') - depends_on('tinyxml2@7.0.0:') - depends_on('toml11@3.6.0:') - depends_on('hdf5@1.10.8~mpi', when='~mpi') - depends_on('hdf5@1.10.8+mpi', when='+mpi') - depends_on('blt') - + depends_on('cmake@3.16', type='build') + depends_on('mpi@3.0:3.1', when='+mpi', type=('build', 'link', 'run')) + depends_on('cxxopts@=3.0.0') + depends_on('fmt@6.0.0:8.0.0+shared') + depends_on('tinyxml2@7.0.0:8.0.0') + depends_on('toml11@3.6:3.7') + depends_on('hdf5@=1.10.8~mpi', when='~mpi') + depends_on('hdf5@=1.10.8+mpi', when='+mpi') + depends_on('googletest@=1.10.0 +gmock') \ No newline at end of file diff --git a/repo/packages/tinyxml2/package.py b/repo/packages/tinyxml2/package.py index c4871ef..0818ef1 100644 --- a/repo/packages/tinyxml2/package.py +++ b/repo/packages/tinyxml2/package.py @@ -25,3 +25,13 @@ class Tinyxml2(CMakePackage): version("2.2.0", sha256="f891224f32e7a06bf279290619cec80cc8ddc335c13696872195ffb87f5bce67") version("2.1.0", sha256="4bdd6569fdce00460bf9cda0ff5dcff46d342b4595900d849cc46a277a74cce6") version("2.0.2", sha256="3cc3aa09cd1ce77736f23488c7cb24e65e11daed4e870ddc8d352aa4070c7c74") + + variant("shared", default=False, description="Build shared library") + + def cmake_args(self): + args = [] + + if self.spec.satisfies("+shared"): + args.append("-DBUILD_SHARED_LIBS=ON") + + return args From 08abb568676d503da4ae98b6703be05e360ab82a Mon Sep 17 00:00:00 2001 From: one Date: Thu, 4 Apr 2024 23:03:31 +0800 Subject: [PATCH 08/73] Change apt sources to BFSU mirrors --- etc/apt/sources.list | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/etc/apt/sources.list b/etc/apt/sources.list index a247bbf..6e3c21a 100644 --- a/etc/apt/sources.list +++ b/etc/apt/sources.list @@ -1,13 +1,14 @@ # 默认注释了源码镜像以提高 apt update 速度，如有需要可自行取消注释 -deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal main restricted universe multiverse -# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal main restricted universe multiverse -deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-updates main restricted universe multiverse -# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-updates main restricted universe multiverse -deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-backports main restricted universe multiverse -# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-backports main restricted universe multiverse -deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-security main restricted universe multiverse -# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-security main restricted universe multiverse +deb https://mirrors.bfsu.edu.cn/ubuntu/ focal main restricted universe multiverse +# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal main restricted universe multiverse +deb https://mirrors.bfsu.edu.cn/ubuntu/ focal-updates main restricted universe multiverse +# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal-updates main restricted universe multiverse +deb https://mirrors.bfsu.edu.cn/ubuntu/ focal-backports main restricted universe multiverse +# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal-backports main restricted universe multiverse + +deb http://security.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse +# deb-src http://security.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse # 预发布软件源，不建议启用 -# deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse -# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse +# deb https://mirrors.bfsu.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse +# # deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse \ No newline at end of file From 5d1212e72ec4f94f1eb29e7786fd54186304b8b4 Mon Sep 17 00:00:00 2001 From: one Date: Thu, 4 Apr 2024 23:04:53 +0800 Subject: [PATCH 09/73] Add back spack mirror commands --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5e37d69..979be11 100644 --- a/Dockerfile +++ b/Dockerfile @@ -114,7 +114,9 @@ RUN deps=(\ "antmoc %$GCC_SPEC ~mpi" \ "antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC" \ "antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC") \ - && for dep in "${deps[@]}"; do spack install --fail-fast -ny $dep; done \ + && for dep in "${deps[@]}"; do \ + spack mirror create -D -d $MIRROR_DIR $dep && \ + spack install --fail-fast -ny $dep; done \ && spack gc -y && spack clean -a \ && spack debug report && spack find -v # Check spack and dependency installation From 7f959b2e23ed0a83f7101a801ad01a5ef22344c0 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 00:52:01 +0800 Subject: [PATCH 10/73] Update spack mirror commands --- .gitignore | 3 +++ Dockerfile | 6 ++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index c18dd8d..d312ee2 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ __pycache__/ +/mirror/ +/ant-moc/ +/antmoc/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 979be11..2a8f897 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,7 +60,7 @@ RUN set -e; \ COPY repo/ $REPO_DIR/ # hold a local package mirror as needed -#COPY mirror/ $MIRROR_DIR/ +COPY mirror/ $MIRROR_DIR/ # set the arch for packages ARG TARGET="x86_64" @@ -114,9 +114,7 @@ RUN deps=(\ "antmoc %$GCC_SPEC ~mpi" \ "antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC" \ "antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC") \ - && for dep in "${deps[@]}"; do \ - spack mirror create -D -d $MIRROR_DIR $dep && \ - spack install --fail-fast -ny $dep; done \ + && for dep in "${deps[@]}"; do spack install --fail-fast -ny $dep; done \ && spack gc -y && spack clean -a \ && spack debug report && spack find -v # Check spack and dependency installation From 806c2288be5c4e95dbd026ae9ea20b3589a6b2b6 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 00:52:13 +0800 Subject: [PATCH 11/73] Update antmoc package --- repo/packages/antmoc/package.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/repo/packages/antmoc/package.py b/repo/packages/antmoc/package.py index 31d6b29..48029ea 100644 --- a/repo/packages/antmoc/package.py +++ b/repo/packages/antmoc/package.py @@ -1,4 +1,4 @@ -from spack import * +from spack.package import * class Antmoc(BundlePackage): @@ -8,15 +8,17 @@ class Antmoc(BundlePackage): maintainers = ['alephpiece'] + license("MIT") + version('0.1.15') variant('mpi', default=False, description='Enable MPI support') - depends_on('cmake@3.16', type='build') + depends_on('cmake@3.16:', type='build') depends_on('mpi@3.0:3.1', when='+mpi', type=('build', 'link', 'run')) - depends_on('cxxopts@=3.0.0') + depends_on('cxxopts@=3.0') depends_on('fmt@6.0.0:8.0.0+shared') - depends_on('tinyxml2@7.0.0:8.0.0') + depends_on('tinyxml2@7.0:9.0+shared') depends_on('toml11@3.6:3.7') depends_on('hdf5@=1.10.8~mpi', when='~mpi') depends_on('hdf5@=1.10.8+mpi', when='+mpi') From e0509588d5c9f8509b981b98f8bda7032c4afaa0 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 00:54:18 +0800 Subject: [PATCH 12/73] Add some scripts for slimtoolkit --- scripts/build-antmoc.sh | 57 +++++++++++++++++++++++++++++++++++++++++ scripts/docker-slim.sh | 13 ++++++++++ scripts/setup-mirror.sh | 21 +++++++++++++++ 3 files changed, 91 insertions(+) create mode 100755 scripts/build-antmoc.sh create mode 100755 scripts/docker-slim.sh create mode 100755 scripts/setup-mirror.sh diff --git a/scripts/build-antmoc.sh b/scripts/build-antmoc.sh new file mode 100755 index 0000000..69c2c2a --- /dev/null +++ b/scripts/build-antmoc.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +sudo -Hi -u root bash << EOF +source /home/hpcer/setup-env.sh +spack uninstall -fy fmt%gcc +spack install -ny googletest@=1.10.0+gmock %gcc +spack install -ny fmt@=8.0.0 %gcc +EOF +sudo -Hi -u hpcer bash << EOF +set -e +whoami + +# Always mount ANT-MOC to this directory in containers +# -v ./ant-moc:/home/hpcer/ant-moc +cd \$HOME/ant-moc/ + +# Setup environment +source ~/setup-env.sh +spack debug report + +# Setup variables +C_COMPILER="gcc" +CXX_COMPILER="g++" +BUILD_TYPE="Release" +BUILD_SHARED_LIBS="ON" +ENABLE_MPI="OFF" +ENABLE_HIP="OFF" +ENABLE_TESTS="ON" +CTEST_RANDOM="ON" +USE_SPECS="antmoc %gcc ~mpi" + +# Build ANT-MOC +spack load cmake%gcc \ +cxxopts%gcc \ +fmt@=8.0.0%gcc \ +tinyxml2%gcc \ +toml11%gcc \ +hdf5~mpi%gcc \ +googletest%gcc +spack find --loaded + +#cmake -S . -B build \ +# -DCMAKE_C_COMPILER=\$C_COMPILER \ +# -DCMAKE_CXX_COMPILER=\$CXX_COMPILER \ +# -DCMAKE_BUILD_TYPE=\$BUILD_TYPE \ +# -DBUILD_SHARED_LIBS:BOOL=\$BUILD_SHARED_LIBS \ +# -DENABLE_TESTS:BOOL=\$ENABLE_TESTS \ +# -DENABLE_MPI:BOOL=\$ENABLE_MPI \ +# -DENABLE_HIP:BOOL=\$ENABLE_HIP +# +#cmake --build build -j +# +#ARGS="--output-on-failure" +#if [ "\$CTEST_RANDOM" == "ON" ]; then ARGS="\$ARGS --schedule-random"; fi +#cd build/ +#ctest \$ARGS + +EOF diff --git a/scripts/docker-slim.sh b/scripts/docker-slim.sh new file mode 100755 index 0000000..981e7a1 --- /dev/null +++ b/scripts/docker-slim.sh @@ -0,0 +1,13 @@ +IMAGE=antmoc/antmoc-ci:0.1.16-alpha + +slim build --target $IMAGE \ + --mount $(pwd)/ant-moc:/home/hpcer/ant-moc \ + --mount $(pwd)/scripts:/home/hpcer/scripts \ + --http-probe=false \ + --show-clogs \ + --include-path /opt/spack \ + --include-path /etc/spack \ + --include-path /opt/software \ + --include-path /opt/repo \ + --entrypoint /bin/bash \ + --exec-file ./scripts/build-antmoc.sh diff --git a/scripts/setup-mirror.sh b/scripts/setup-mirror.sh new file mode 100755 index 0000000..a5ced29 --- /dev/null +++ b/scripts/setup-mirror.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Please make sure you are in the root directory of this git repo +set -e +MIRROR_DIR=$(pwd)/mirror +[ -d $MIRROR_DIR ] || mkdir $MIRROR_DIR + +# Copied from Dockerfile +GCC_SPEC="gcc" +CLANG_SPEC="clang" +MPICH_SPEC="mpich~fortran" +OPENMPI_SPEC="openmpi" + +deps=(\ + "antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC" \ + "antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC" \ + "antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC") + +for dep in "${deps[@]}"; do + spack mirror create -D -d $MIRROR_DIR $dep +done From 8bdf5e648561e174b6bf67a15a4efdf6b8e85732 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 00:56:44 +0800 Subject: [PATCH 13/73] Fix antmoc package --- repo/packages/antmoc/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo/packages/antmoc/package.py b/repo/packages/antmoc/package.py index 48029ea..ce66dc3 100644 --- a/repo/packages/antmoc/package.py +++ b/repo/packages/antmoc/package.py @@ -16,7 +16,7 @@ class Antmoc(BundlePackage): depends_on('cmake@3.16:', type='build') depends_on('mpi@3.0:3.1', when='+mpi', type=('build', 'link', 'run')) - depends_on('cxxopts@=3.0') + depends_on('cxxopts@=3.0.0') depends_on('fmt@6.0.0:8.0.0+shared') depends_on('tinyxml2@7.0:9.0+shared') depends_on('toml11@3.6:3.7') From bf50deb8d482aadab151c512ce9dffae03560a67 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 01:24:08 +0800 Subject: [PATCH 14/73] Update build-antmoc.sh --- scripts/build-antmoc.sh | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/scripts/build-antmoc.sh b/scripts/build-antmoc.sh index 69c2c2a..66c7645 100755 --- a/scripts/build-antmoc.sh +++ b/scripts/build-antmoc.sh @@ -1,12 +1,7 @@ #!/usr/bin/env bash -sudo -Hi -u root bash << EOF -source /home/hpcer/setup-env.sh -spack uninstall -fy fmt%gcc -spack install -ny googletest@=1.10.0+gmock %gcc -spack install -ny fmt@=8.0.0 %gcc -EOF sudo -Hi -u hpcer bash << EOF set -e +source ~/.bashrc whoami # Always mount ANT-MOC to this directory in containers @@ -29,13 +24,7 @@ CTEST_RANDOM="ON" USE_SPECS="antmoc %gcc ~mpi" # Build ANT-MOC -spack load cmake%gcc \ -cxxopts%gcc \ -fmt@=8.0.0%gcc \ -tinyxml2%gcc \ -toml11%gcc \ -hdf5~mpi%gcc \ -googletest%gcc +spack load cmake%gcc \$USE_SPECS spack find --loaded #cmake -S . -B build \ From 5c05bb7137dab5cc0e96e76496c93c56c0291b9e Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 01:35:16 +0800 Subject: [PATCH 15/73] Add -j to spack install --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2a8f897..5539a79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -114,7 +114,7 @@ RUN deps=(\ "antmoc %$GCC_SPEC ~mpi" \ "antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC" \ "antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC") \ - && for dep in "${deps[@]}"; do spack install --fail-fast -ny $dep; done \ + && for dep in "${deps[@]}"; do spack install -j $(nproc) --fail-fast -ny $dep; done \ && spack gc -y && spack clean -a \ && spack debug report && spack find -v # Check spack and dependency installation From 40fcaf0b908668a3fb849610af794b32f4327dfe Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 12:12:03 +0800 Subject: [PATCH 16/73] Update system version to ubuntu-jammy --- Dockerfile | 8 ++++---- Makefile | 2 +- etc/apt/sources.list | 20 ++++++++++---------- scripts/setup-mirror.sh | 4 +++- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5539a79..cb30f80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ ARG SPACK_VERSION="0.21.2" -ARG SPACK_IMAGE="spack/ubuntu-focal" +ARG SPACK_IMAGE="spack/ubuntu-jammy" FROM ${SPACK_IMAGE}:${SPACK_VERSION} AS builder ARG UBUNTU_CODE -ENV UBUNTU_CODE=${UBUNTU_CODE:-"focal"} +ENV UBUNTU_CODE=${UBUNTU_CODE:-"jammy"} LABEL maintainer="An Wang " @@ -19,7 +19,7 @@ ARG AMDGPU_VERSION=5.3 # install LLVM and CMake for spack, and # install ROCm HIP, see https://github.com/ROCm/ROCm-docker/blob/master/dev/Dockerfile-ubuntu-20.04 COPY etc/apt/ /etc/apt/ -RUN sed -i -e "s/focal/$UBUNTU_CODE/g" /etc/apt/sources.list \ +RUN sed -i -e "s/jammy/$UBUNTU_CODE/g" /etc/apt/sources.list \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl libnuma-dev gnupg \ && curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - \ @@ -35,7 +35,7 @@ RUN sed -i -e "s/focal/$UBUNTU_CODE/g" /etc/apt/sources.list \ python3-pip \ rocm-dev \ build-essential \ - llvm-10 clang-10 libomp-10-dev cmake openssh-server && \ + llvm-15 clang-15 libomp-15-dev cmake openssh-server && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/Makefile b/Makefile index bba278e..7de34d5 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ #=============================================================================== # Build-time arguments -UBUNTU_CODE ?= "focal" +UBUNTU_CODE ?= "jammy" SPACK_VERSION ?= "0.21.2" SPACK_IMAGE = "spack/ubuntu-$(UBUNTU_CODE)" diff --git a/etc/apt/sources.list b/etc/apt/sources.list index 6e3c21a..19e07f1 100644 --- a/etc/apt/sources.list +++ b/etc/apt/sources.list @@ -1,14 +1,14 @@ # 默认注释了源码镜像以提高 apt update 速度，如有需要可自行取消注释 -deb https://mirrors.bfsu.edu.cn/ubuntu/ focal main restricted universe multiverse -# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal main restricted universe multiverse -deb https://mirrors.bfsu.edu.cn/ubuntu/ focal-updates main restricted universe multiverse -# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal-updates main restricted universe multiverse -deb https://mirrors.bfsu.edu.cn/ubuntu/ focal-backports main restricted universe multiverse -# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal-backports main restricted universe multiverse +deb https://mirrors.bfsu.edu.cn/ubuntu/ jammy main restricted universe multiverse +# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ jammy main restricted universe multiverse +deb https://mirrors.bfsu.edu.cn/ubuntu/ jammy-updates main restricted universe multiverse +# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ jammy-updates main restricted universe multiverse +deb https://mirrors.bfsu.edu.cn/ubuntu/ jammy-backports main restricted universe multiverse +# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ jammy-backports main restricted universe multiverse -deb http://security.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse -# deb-src http://security.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse +deb http://security.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse +# deb-src http://security.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse # 预发布软件源，不建议启用 -# deb https://mirrors.bfsu.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse -# # deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse \ No newline at end of file +# deb https://mirrors.bfsu.edu.cn/ubuntu/ jammy-proposed main restricted universe multiverse +# # deb-src https://mirrors.bfsu.edu.cn/ubuntu/ jammy-proposed main restricted universe multiverse \ No newline at end of file diff --git a/scripts/setup-mirror.sh b/scripts/setup-mirror.sh index a5ced29..fe69fc2 100755 --- a/scripts/setup-mirror.sh +++ b/scripts/setup-mirror.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash # Please make sure you are in the root directory of this git repo -set -e MIRROR_DIR=$(pwd)/mirror [ -d $MIRROR_DIR ] || mkdir $MIRROR_DIR +# Add a self-hosted repo +spack repo add repo/ + # Copied from Dockerfile GCC_SPEC="gcc" CLANG_SPEC="clang" From e3b006363cfbb7ba9b7954ccd37cdd1735e29ad8 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 13:13:59 +0800 Subject: [PATCH 17/73] Failed to install ROCm-dev on Ubuntu jammy, rolling back --- Dockerfile | 12 ++++++------ Makefile | 10 ++++++---- etc/apt/sources.list | 20 ++++++++++---------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index cb30f80..d4fa5ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ ARG SPACK_VERSION="0.21.2" -ARG SPACK_IMAGE="spack/ubuntu-jammy" +ARG SPACK_IMAGE="spack/ubuntu-focal" FROM ${SPACK_IMAGE}:${SPACK_VERSION} AS builder ARG UBUNTU_CODE -ENV UBUNTU_CODE=${UBUNTU_CODE:-"jammy"} +ENV UBUNTU_CODE=${UBUNTU_CODE:-"focal"} LABEL maintainer="An Wang " @@ -13,13 +13,13 @@ WORKDIR /tmp # Install system compilers and libraries #------------------------------------------------------------------------------- # Register the ROCM package repository, and install rocm-dev package -ARG ROCM_VERSION=5.3 -ARG AMDGPU_VERSION=5.3 +ARG ROCM_VERSION=5.4.6 +ARG AMDGPU_VERSION=5.4.6 # install LLVM and CMake for spack, and # install ROCm HIP, see https://github.com/ROCm/ROCm-docker/blob/master/dev/Dockerfile-ubuntu-20.04 COPY etc/apt/ /etc/apt/ -RUN sed -i -e "s/jammy/$UBUNTU_CODE/g" /etc/apt/sources.list \ +RUN sed -i -e "s/focal/$UBUNTU_CODE/g" /etc/apt/sources.list \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl libnuma-dev gnupg \ && curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - \ @@ -35,7 +35,7 @@ RUN sed -i -e "s/jammy/$UBUNTU_CODE/g" /etc/apt/sources.list \ python3-pip \ rocm-dev \ build-essential \ - llvm-15 clang-15 libomp-15-dev cmake openssh-server && \ + llvm-12 clang-12 libomp-12-dev cmake openssh-server && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/Makefile b/Makefile index 7de34d5..6a71baf 100644 --- a/Makefile +++ b/Makefile @@ -3,12 +3,14 @@ #=============================================================================== # Build-time arguments -UBUNTU_CODE ?= "jammy" -SPACK_VERSION ?= "0.21.2" -SPACK_IMAGE = "spack/ubuntu-$(UBUNTU_CODE)" +UBUNTU_CODE ?= focal +SPACK_VERSION ?= 0.21.2 +SPACK_IMAGE = spack/ubuntu-$(UBUNTU_CODE) +ROCM_VERSION ?= 5.4.6 +AMDGPU_VERSION ?= 5.4.6 # Target -TARGET ?= "x86_64" +TARGET ?= x86_64 # Image name DOCKER_IMAGE ?= antmoc/antmoc-ci diff --git a/etc/apt/sources.list b/etc/apt/sources.list index 19e07f1..6e3c21a 100644 --- a/etc/apt/sources.list +++ b/etc/apt/sources.list @@ -1,14 +1,14 @@ # 默认注释了源码镜像以提高 apt update 速度，如有需要可自行取消注释 -deb https://mirrors.bfsu.edu.cn/ubuntu/ jammy main restricted universe multiverse -# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ jammy main restricted universe multiverse -deb https://mirrors.bfsu.edu.cn/ubuntu/ jammy-updates main restricted universe multiverse -# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ jammy-updates main restricted universe multiverse -deb https://mirrors.bfsu.edu.cn/ubuntu/ jammy-backports main restricted universe multiverse -# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ jammy-backports main restricted universe multiverse +deb https://mirrors.bfsu.edu.cn/ubuntu/ focal main restricted universe multiverse +# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal main restricted universe multiverse +deb https://mirrors.bfsu.edu.cn/ubuntu/ focal-updates main restricted universe multiverse +# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal-updates main restricted universe multiverse +deb https://mirrors.bfsu.edu.cn/ubuntu/ focal-backports main restricted universe multiverse +# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal-backports main restricted universe multiverse -deb http://security.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse -# deb-src http://security.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse +deb http://security.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse +# deb-src http://security.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse # 预发布软件源，不建议启用 -# deb https://mirrors.bfsu.edu.cn/ubuntu/ jammy-proposed main restricted universe multiverse -# # deb-src https://mirrors.bfsu.edu.cn/ubuntu/ jammy-proposed main restricted universe multiverse \ No newline at end of file +# deb https://mirrors.bfsu.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse +# # deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse \ No newline at end of file From dd561f0840edff4ed2687766602970f9ae5a132c Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 13:44:10 +0800 Subject: [PATCH 18/73] Migrate some system packages to spack --- Dockerfile | 9 ++------- scripts/build-antmoc.sh | 30 +++++++++++++++--------------- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index d4fa5ed..b59d6cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ RUN sed -i -e "s/focal/$UBUNTU_CODE/g" /etc/apt/sources.list \ python3-pip \ rocm-dev \ build-essential \ - llvm-12 clang-12 libomp-12-dev cmake openssh-server && \ + llvm-12 clang-12 libomp-12-dev openssh-server && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -88,11 +88,7 @@ RUN spack compiler find \ && spack external find --scope system --not-buildable \ gcc \ llvm \ - autoconf \ - automake \ - cmake \ - gmake \ - libtool \ + curl \ perl #------------------------------------------------------------------------------- @@ -107,7 +103,6 @@ ARG MPICH_SPEC="mpich~fortran" ARG OPENMPI_SPEC="openmpi" RUN deps=(\ - "cmake %$GCC_SPEC" \ "lcov@=2.0 %$GCC_SPEC" \ "antmoc %$CLANG_SPEC ~mpi" \ "antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC" \ diff --git a/scripts/build-antmoc.sh b/scripts/build-antmoc.sh index 66c7645..505a383 100755 --- a/scripts/build-antmoc.sh +++ b/scripts/build-antmoc.sh @@ -27,20 +27,20 @@ USE_SPECS="antmoc %gcc ~mpi" spack load cmake%gcc \$USE_SPECS spack find --loaded -#cmake -S . -B build \ -# -DCMAKE_C_COMPILER=\$C_COMPILER \ -# -DCMAKE_CXX_COMPILER=\$CXX_COMPILER \ -# -DCMAKE_BUILD_TYPE=\$BUILD_TYPE \ -# -DBUILD_SHARED_LIBS:BOOL=\$BUILD_SHARED_LIBS \ -# -DENABLE_TESTS:BOOL=\$ENABLE_TESTS \ -# -DENABLE_MPI:BOOL=\$ENABLE_MPI \ -# -DENABLE_HIP:BOOL=\$ENABLE_HIP -# -#cmake --build build -j -# -#ARGS="--output-on-failure" -#if [ "\$CTEST_RANDOM" == "ON" ]; then ARGS="\$ARGS --schedule-random"; fi -#cd build/ -#ctest \$ARGS +cmake -S . -B build \ + -DCMAKE_C_COMPILER=\$C_COMPILER \ + -DCMAKE_CXX_COMPILER=\$CXX_COMPILER \ + -DCMAKE_BUILD_TYPE=\$BUILD_TYPE \ + -DBUILD_SHARED_LIBS:BOOL=\$BUILD_SHARED_LIBS \ + -DENABLE_TESTS:BOOL=\$ENABLE_TESTS \ + -DENABLE_MPI:BOOL=\$ENABLE_MPI \ + -DENABLE_HIP:BOOL=\$ENABLE_HIP + +cmake --build build -j\$(nproc) -v + +ARGS="--output-on-failure" +if [ "\$CTEST_RANDOM" == "ON" ]; then ARGS="\$ARGS --schedule-random"; fi +cd build/ +ctest \$ARGS EOF From 613fe231583038ee3f7afec301d76ac75cead7fd Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 13:45:20 +0800 Subject: [PATCH 19/73] Update github actions --- .github/workflows/dockerimage.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index c9b5afa..8ac1c63 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -4,7 +4,6 @@ on: push: tags: - 'v*' - pull_request: jobs: main: From d2e07e7b9a242ca3626e1fbaffcce1b5030ea6fe Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 13:46:46 +0800 Subject: [PATCH 20/73] Specify cmake version --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index b59d6cf..5389161 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,6 +103,7 @@ ARG MPICH_SPEC="mpich~fortran" ARG OPENMPI_SPEC="openmpi" RUN deps=(\ + "cmake@3.24 %GCC_SPEC" \ "lcov@=2.0 %$GCC_SPEC" \ "antmoc %$CLANG_SPEC ~mpi" \ "antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC" \ From 40647498beb7998672a0286f6c7f3474b867f71f Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 13:53:17 +0800 Subject: [PATCH 21/73] Remove cmake from spack install --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5389161..b59d6cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,7 +103,6 @@ ARG MPICH_SPEC="mpich~fortran" ARG OPENMPI_SPEC="openmpi" RUN deps=(\ - "cmake@3.24 %GCC_SPEC" \ "lcov@=2.0 %$GCC_SPEC" \ "antmoc %$CLANG_SPEC ~mpi" \ "antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC" \ From a1a217c0507a7e0a40ea9dbc14262fbbe0bd3441 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 14:14:08 +0800 Subject: [PATCH 22/73] Add system packages back --- Dockerfile | 7 ++++++- scripts/build-antmoc.sh | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b59d6cf..897b975 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ RUN sed -i -e "s/focal/$UBUNTU_CODE/g" /etc/apt/sources.list \ python3-pip \ rocm-dev \ build-essential \ - llvm-12 clang-12 libomp-12-dev openssh-server && \ + llvm-12 clang-12 libomp-12-dev cmake openssh-server && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -88,6 +88,11 @@ RUN spack compiler find \ && spack external find --scope system --not-buildable \ gcc \ llvm \ + autoconf \ + automake \ + cmake \ + gmake \ + libtool \ curl \ perl diff --git a/scripts/build-antmoc.sh b/scripts/build-antmoc.sh index 505a383..f730106 100755 --- a/scripts/build-antmoc.sh +++ b/scripts/build-antmoc.sh @@ -27,6 +27,8 @@ USE_SPECS="antmoc %gcc ~mpi" spack load cmake%gcc \$USE_SPECS spack find --loaded +rm -rf build/ &> /dev/null + cmake -S . -B build \ -DCMAKE_C_COMPILER=\$C_COMPILER \ -DCMAKE_CXX_COMPILER=\$CXX_COMPILER \ From a1f5c54b418a6f97ab2edeea58bb7f08d3042a54 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 16:01:49 +0800 Subject: [PATCH 23/73] Fix cmake package --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 897b975..221975d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,6 +108,7 @@ ARG MPICH_SPEC="mpich~fortran" ARG OPENMPI_SPEC="openmpi" RUN deps=(\ + "cmake %$GCC_SPEC" \ "lcov@=2.0 %$GCC_SPEC" \ "antmoc %$CLANG_SPEC ~mpi" \ "antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC" \ From 30c8e5d47a4c32816c48dbed20066d6196d5a530 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 16:14:13 +0800 Subject: [PATCH 24/73] Remove curl from spack external --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 221975d..be0de2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -93,7 +93,6 @@ RUN spack compiler find \ cmake \ gmake \ libtool \ - curl \ perl #------------------------------------------------------------------------------- From 8bad0ca99945d6190702396341c9ffb97ce712ed Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 16:37:42 +0800 Subject: [PATCH 25/73] Update system version to jammy --- Dockerfile | 14 ++++++++------ Makefile | 2 +- etc/apt/sources.list | 20 ++++++++++---------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index be0de2c..1626018 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ ARG SPACK_VERSION="0.21.2" -ARG SPACK_IMAGE="spack/ubuntu-focal" +ARG SPACK_IMAGE="spack/ubuntu-jammy" FROM ${SPACK_IMAGE}:${SPACK_VERSION} AS builder ARG UBUNTU_CODE -ENV UBUNTU_CODE=${UBUNTU_CODE:-"focal"} +ENV UBUNTU_CODE=${UBUNTU_CODE:-"jammy"} LABEL maintainer="An Wang " @@ -17,13 +17,15 @@ ARG ROCM_VERSION=5.4.6 ARG AMDGPU_VERSION=5.4.6 # install LLVM and CMake for spack, and -# install ROCm HIP, see https://github.com/ROCm/ROCm-docker/blob/master/dev/Dockerfile-ubuntu-20.04 +# install ROCm HIP, see https://github.com/ROCm/ROCm-docker/blob/master/dev/Dockerfile-ubuntu-22.04 COPY etc/apt/ /etc/apt/ -RUN sed -i -e "s/focal/$UBUNTU_CODE/g" /etc/apt/sources.list \ +ARG APT_PREF="Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600" +RUN sed -i -e "s/jammy/$UBUNTU_CODE/g" /etc/apt/sources.list \ + && echo -e "$APT_PREF" | tee /etc/apt/preferences.d/rocm-pin-600 \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl libnuma-dev gnupg \ && curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - \ - && printf "deb [arch=amd64] https://repo.radeon.com/rocm/apt/$ROCM_VERSION/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list \ + && printf "deb [arch=amd64] https://repo.radeon.com/rocm/apt/$ROCM_VERSION/ $UBUNTU_CODE main" | tee /etc/apt/sources.list.d/rocm.list \ && printf "deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu $UBUNTU_CODE main" | tee /etc/apt/sources.list.d/amdgpu.list \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ @@ -35,7 +37,7 @@ RUN sed -i -e "s/focal/$UBUNTU_CODE/g" /etc/apt/sources.list \ python3-pip \ rocm-dev \ build-essential \ - llvm-12 clang-12 libomp-12-dev cmake openssh-server && \ + llvm-14 clang-14 libomp-14-dev cmake openssh-server && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/Makefile b/Makefile index 6a71baf..ee06dce 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ #=============================================================================== # Build-time arguments -UBUNTU_CODE ?= focal +UBUNTU_CODE ?= jammy SPACK_VERSION ?= 0.21.2 SPACK_IMAGE = spack/ubuntu-$(UBUNTU_CODE) ROCM_VERSION ?= 5.4.6 diff --git a/etc/apt/sources.list b/etc/apt/sources.list index 6e3c21a..19e07f1 100644 --- a/etc/apt/sources.list +++ b/etc/apt/sources.list @@ -1,14 +1,14 @@ # 默认注释了源码镜像以提高 apt update 速度，如有需要可自行取消注释 -deb https://mirrors.bfsu.edu.cn/ubuntu/ focal main restricted universe multiverse -# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal main restricted universe multiverse -deb https://mirrors.bfsu.edu.cn/ubuntu/ focal-updates main restricted universe multiverse -# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal-updates main restricted universe multiverse -deb https://mirrors.bfsu.edu.cn/ubuntu/ focal-backports main restricted universe multiverse -# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal-backports main restricted universe multiverse +deb https://mirrors.bfsu.edu.cn/ubuntu/ jammy main restricted universe multiverse +# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ jammy main restricted universe multiverse +deb https://mirrors.bfsu.edu.cn/ubuntu/ jammy-updates main restricted universe multiverse +# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ jammy-updates main restricted universe multiverse +deb https://mirrors.bfsu.edu.cn/ubuntu/ jammy-backports main restricted universe multiverse +# deb-src https://mirrors.bfsu.edu.cn/ubuntu/ jammy-backports main restricted universe multiverse -deb http://security.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse -# deb-src http://security.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse +deb http://security.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse +# deb-src http://security.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse # 预发布软件源，不建议启用 -# deb https://mirrors.bfsu.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse -# # deb-src https://mirrors.bfsu.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse \ No newline at end of file +# deb https://mirrors.bfsu.edu.cn/ubuntu/ jammy-proposed main restricted universe multiverse +# # deb-src https://mirrors.bfsu.edu.cn/ubuntu/ jammy-proposed main restricted universe multiverse \ No newline at end of file From e9250938014099fc954fbaf062e1bf9aa0519678 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 17:01:51 +0800 Subject: [PATCH 26/73] Update googletest version --- repo/packages/antmoc/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo/packages/antmoc/package.py b/repo/packages/antmoc/package.py index ce66dc3..6b33b88 100644 --- a/repo/packages/antmoc/package.py +++ b/repo/packages/antmoc/package.py @@ -22,4 +22,4 @@ class Antmoc(BundlePackage): depends_on('toml11@3.6:3.7') depends_on('hdf5@=1.10.8~mpi', when='~mpi') depends_on('hdf5@=1.10.8+mpi', when='+mpi') - depends_on('googletest@=1.10.0 +gmock') \ No newline at end of file + depends_on('googletest@1.10:1.12 +gmock') \ No newline at end of file From 2df3a98912fc58e33f6735cd7ec39122c265592a Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 17:12:23 +0800 Subject: [PATCH 27/73] Add rocthrust back --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 1626018..3cc29ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,7 @@ RUN sed -i -e "s/jammy/$UBUNTU_CODE/g" /etc/apt/sources.list \ python3 \ python3-pip \ rocm-dev \ + rocthrust-dev \ build-essential \ llvm-14 clang-14 libomp-14-dev cmake openssh-server && \ apt-get clean && \ From 696f3fe32b5160cfa9ff288f249d9be02a6c7faa Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 17:18:48 +0800 Subject: [PATCH 28/73] Split apt install commands --- Dockerfile | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3cc29ca..b2983cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,10 +37,16 @@ RUN sed -i -e "s/jammy/$UBUNTU_CODE/g" /etc/apt/sources.list \ python3-pip \ rocm-dev \ rocthrust-dev \ - build-essential \ - llvm-14 clang-14 libomp-14-dev cmake openssh-server && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* + build-essential + +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + llvm-14 \ + clang-14 \ + libomp-14-dev \ + cmake \ + openssh-server \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* #------------------------------------------------------------------------------- # Set up spack From 02d99faeb9043d36e7dc902a127f517ab3cb93ee Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 17:36:53 +0800 Subject: [PATCH 29/73] Update comments --- Dockerfile | 3 +-- scripts/build-antmoc.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b2983cf..ae8450f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,6 @@ WORKDIR /tmp ARG ROCM_VERSION=5.4.6 ARG AMDGPU_VERSION=5.4.6 -# install LLVM and CMake for spack, and # install ROCm HIP, see https://github.com/ROCm/ROCm-docker/blob/master/dev/Dockerfile-ubuntu-22.04 COPY etc/apt/ /etc/apt/ ARG APT_PREF="Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600" @@ -38,7 +37,7 @@ RUN sed -i -e "s/jammy/$UBUNTU_CODE/g" /etc/apt/sources.list \ rocm-dev \ rocthrust-dev \ build-essential - +# install LLVM and CMake for spack RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ llvm-14 \ clang-14 \ diff --git a/scripts/build-antmoc.sh b/scripts/build-antmoc.sh index f730106..43dca67 100755 --- a/scripts/build-antmoc.sh +++ b/scripts/build-antmoc.sh @@ -38,7 +38,7 @@ cmake -S . -B build \ -DENABLE_MPI:BOOL=\$ENABLE_MPI \ -DENABLE_HIP:BOOL=\$ENABLE_HIP -cmake --build build -j\$(nproc) -v +cmake --build build -j\$(nproc) ARGS="--output-on-failure" if [ "\$CTEST_RANDOM" == "ON" ]; then ARGS="\$ARGS --schedule-random"; fi From 9149cdea51993188abada8d3c23c432e3938ddd3 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 19:05:55 +0800 Subject: [PATCH 30/73] Add tests --- Dockerfile | 5 +- scripts/build-antmoc.sh | 111 +++++++++++++++++++++++++++------------- 2 files changed, 80 insertions(+), 36 deletions(-) diff --git a/Dockerfile b/Dockerfile index ae8450f..7e3b2a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,10 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ cmake \ openssh-server \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && rm -f /usr/bin/clang /usr/bin/clang++ &> /dev/null \ + && ln -s /usr/bin/clang-14 /usr/bin/clang \ + && ln -s /usr/bin/clang++-14 /usr/bin/clang++ #------------------------------------------------------------------------------- # Set up spack diff --git a/scripts/build-antmoc.sh b/scripts/build-antmoc.sh index 43dca67..87832e7 100755 --- a/scripts/build-antmoc.sh +++ b/scripts/build-antmoc.sh @@ -1,7 +1,15 @@ #!/usr/bin/env bash +# For slim toolkit 1.40.11 + +sudo -Hi -u root bash << EOF +rm -f /usr/bin/clang /usr/bin/clang++ +ln -s /usr/bin/clang-14 /usr/bin/clang +ln -s /usr/bin/clang++-14 /usr/bin/clang++ +EOF + sudo -Hi -u hpcer bash << EOF +# Compilers in tuple (C compiler, C++ compiler, spack spec) set -e -source ~/.bashrc whoami # Always mount ANT-MOC to this directory in containers @@ -9,40 +17,73 @@ whoami cd \$HOME/ant-moc/ # Setup environment -source ~/setup-env.sh +source \$HOME/setup-env.sh spack debug report -# Setup variables -C_COMPILER="gcc" -CXX_COMPILER="g++" -BUILD_TYPE="Release" -BUILD_SHARED_LIBS="ON" -ENABLE_MPI="OFF" -ENABLE_HIP="OFF" -ENABLE_TESTS="ON" -CTEST_RANDOM="ON" -USE_SPECS="antmoc %gcc ~mpi" - -# Build ANT-MOC -spack load cmake%gcc \$USE_SPECS -spack find --loaded - -rm -rf build/ &> /dev/null - -cmake -S . -B build \ - -DCMAKE_C_COMPILER=\$C_COMPILER \ - -DCMAKE_CXX_COMPILER=\$CXX_COMPILER \ - -DCMAKE_BUILD_TYPE=\$BUILD_TYPE \ - -DBUILD_SHARED_LIBS:BOOL=\$BUILD_SHARED_LIBS \ - -DENABLE_TESTS:BOOL=\$ENABLE_TESTS \ - -DENABLE_MPI:BOOL=\$ENABLE_MPI \ - -DENABLE_HIP:BOOL=\$ENABLE_HIP - -cmake --build build -j\$(nproc) - -ARGS="--output-on-failure" -if [ "\$CTEST_RANDOM" == "ON" ]; then ARGS="\$ARGS --schedule-random"; fi -cd build/ -ctest \$ARGS +# Run tests +declare -A CCs=( + ["gcc"]="gcc g++ %gcc" \ + ["clang"]="clang clang++ %clang" \ + ["hipcc"]="hipcc hipcc %gcc") -EOF +# MPI specs +declare -A MPIs=( \ + ["serial"]="~mpi" \ + ["mpich"]="+mpi^mpich" \ + ["openmpi"]="+mpi^openmpi") + +for cc_id in "\${!CCs[@]}"; do + declare -a cc=(\${CCs[\$cc_id]}) + for mpi in "\${!MPIs[@]}"; do + C_COMPILER="\${cc[0]}" + CXX_COMPILER="\${cc[1]}" + USE_SPECS="antmoc \${cc[2]} \${MPIs[\$mpi]}" + BUILD_TYPE=Release + BUILD_SHARED_LIBS=ON + ENABLE_TESTS=ON + + # Enable MPI or not + if [ \$mpi == "serial" ]; then + ENABLE_MPI=OFF + CTEST_RANDOM=ON + else + ENABLE_MPI=ON + CTEST_RANDOM=OFF + fi + + # Enable HIP or not + if [ \${cc[0]} == "hipcc" ]; then + ENABLE_HIP=ON + else + ENABLE_HIP=OFF + fi + + # Load dependencies + spack load cmake%gcc \$USE_SPECS + spack find --loaded + + rm -rf build/ &> /dev/null + + # Build ANT-MOC + cmake -S . -B build \ + -DCMAKE_C_COMPILER=\$C_COMPILER \ + -DCMAKE_CXX_COMPILER=\$CXX_COMPILER \ + -DCMAKE_BUILD_TYPE=\$BUILD_TYPE \ + -DBUILD_SHARED_LIBS:BOOL=\$BUILD_SHARED_LIBS \ + -DENABLE_TESTS:BOOL=\$ENABLE_TESTS \ + -DENABLE_MPI:BOOL=\$ENABLE_MPI \ + -DENABLE_HIP:BOOL=\$ENABLE_HIP + + cmake --build build -j\$(nproc) + + if [ ! \${cc[0]} == "hipcc" ]; then + ARGS="--output-on-failure" + if [ "\$CTEST_RANDOM" == "ON" ]; then ARGS="\$ARGS --schedule-random"; fi + cd build/ + # Exclude broken tests + # FIXME: these tests are broken on Ubuntu jammy but work on Ubuntu focal + ctest \$ARGS -E unit_test_initialize* + fi + done +done +EOF \ No newline at end of file From e74774bd024db29f50b93026e48c722c487b7a01 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 21:51:22 +0800 Subject: [PATCH 31/73] Update build-antmoc.sh --- repo/packages/antmoc/package.py | 2 +- scripts/build-antmoc.sh | 142 +++++++++++++++++++------------- scripts/docker-slim.sh | 3 +- 3 files changed, 85 insertions(+), 62 deletions(-) diff --git a/repo/packages/antmoc/package.py b/repo/packages/antmoc/package.py index 6b33b88..9898973 100644 --- a/repo/packages/antmoc/package.py +++ b/repo/packages/antmoc/package.py @@ -15,7 +15,7 @@ class Antmoc(BundlePackage): variant('mpi', default=False, description='Enable MPI support') depends_on('cmake@3.16:', type='build') - depends_on('mpi@3.0:3.1', when='+mpi', type=('build', 'link', 'run')) + depends_on('mpi@=3.0', when='+mpi', type=('build', 'link', 'run')) depends_on('cxxopts@=3.0.0') depends_on('fmt@6.0.0:8.0.0+shared') depends_on('tinyxml2@7.0:9.0+shared') diff --git a/scripts/build-antmoc.sh b/scripts/build-antmoc.sh index 87832e7..1e14607 100755 --- a/scripts/build-antmoc.sh +++ b/scripts/build-antmoc.sh @@ -8,82 +8,106 @@ ln -s /usr/bin/clang++-14 /usr/bin/clang++ EOF sudo -Hi -u hpcer bash << EOF -# Compilers in tuple (C compiler, C++ compiler, spack spec) set -e whoami +WORKDIR=\$HOME/ant-moc + # Always mount ANT-MOC to this directory in containers -# -v ./ant-moc:/home/hpcer/ant-moc -cd \$HOME/ant-moc/ +# -v ./ant-moc:/opt/mnt/ant-moc +cp -r /opt/mnt/ant-moc \$WORKDIR +cd \$WORKDIR + +# Change permissions +ORIGIN_OWNER=\$(stat -c '%u:%g' tests) +sudo chown -R hpcer:hpcer tests # Setup environment source \$HOME/setup-env.sh spack debug report +spack find -v -# Run tests -declare -A CCs=( +# Compilers in tuple (C compiler, C++ compiler, spack spec) +declare -A COMPILERS=( \ ["gcc"]="gcc g++ %gcc" \ ["clang"]="clang clang++ %clang" \ ["hipcc"]="hipcc hipcc %gcc") # MPI specs -declare -A MPIs=( \ +declare -A MPIS=( \ ["serial"]="~mpi" \ ["mpich"]="+mpi^mpich" \ ["openmpi"]="+mpi^openmpi") -for cc_id in "\${!CCs[@]}"; do - declare -a cc=(\${CCs[\$cc_id]}) - for mpi in "\${!MPIs[@]}"; do - C_COMPILER="\${cc[0]}" - CXX_COMPILER="\${cc[1]}" - USE_SPECS="antmoc \${cc[2]} \${MPIs[\$mpi]}" - BUILD_TYPE=Release - BUILD_SHARED_LIBS=ON - ENABLE_TESTS=ON - - # Enable MPI or not - if [ \$mpi == "serial" ]; then - ENABLE_MPI=OFF - CTEST_RANDOM=ON - else - ENABLE_MPI=ON - CTEST_RANDOM=OFF - fi - - # Enable HIP or not - if [ \${cc[0]} == "hipcc" ]; then - ENABLE_HIP=ON - else - ENABLE_HIP=OFF - fi - - # Load dependencies - spack load cmake%gcc \$USE_SPECS - spack find --loaded - - rm -rf build/ &> /dev/null - - # Build ANT-MOC - cmake -S . -B build \ - -DCMAKE_C_COMPILER=\$C_COMPILER \ - -DCMAKE_CXX_COMPILER=\$CXX_COMPILER \ - -DCMAKE_BUILD_TYPE=\$BUILD_TYPE \ - -DBUILD_SHARED_LIBS:BOOL=\$BUILD_SHARED_LIBS \ - -DENABLE_TESTS:BOOL=\$ENABLE_TESTS \ - -DENABLE_MPI:BOOL=\$ENABLE_MPI \ - -DENABLE_HIP:BOOL=\$ENABLE_HIP - - cmake --build build -j\$(nproc) - - if [ ! \${cc[0]} == "hipcc" ]; then - ARGS="--output-on-failure" - if [ "\$CTEST_RANDOM" == "ON" ]; then ARGS="\$ARGS --schedule-random"; fi - cd build/ - # Exclude broken tests - # FIXME: these tests are broken on Ubuntu jammy but work on Ubuntu focal - ctest \$ARGS -E unit_test_initialize* - fi - done +# Test cases +declare -a TESTS=( \ + "gcc serial run" "gcc mpich run" "gcc openmpi run" \ + "clang serial run" "clang mpich run" \ + "hipcc serial build" "hipcc mpich build" "hipcc openmpi build") + +# Run tests +for s in "\${TESTS[@]}"; do + cd \$WORKDIR + + declare -a test=(\${s[@]}) + declare -a cc=(\${COMPILERS[\${test[0]}]}) + mpi=\${test[1]} + stage=\${test[2]} + + echo "\${cc[@]} \$mpi \$stage" + + C_COMPILER="\${cc[0]}" + CXX_COMPILER="\${cc[1]}" + USE_SPECS="antmoc \${cc[2]} \${MPIS[\$mpi]}" + BUILD_TYPE=Release + BUILD_SHARED_LIBS=ON + ENABLE_TESTS=ON + + # Enable MPI or not + if [ \$mpi == "serial" ]; then + ENABLE_MPI=OFF + CTEST_RANDOM=ON + else + ENABLE_MPI=ON + CTEST_RANDOM=OFF + fi + + # Enable HIP or not + if [ \${cc[0]} == "hipcc" ]; then + ENABLE_HIP=ON + else + ENABLE_HIP=OFF + fi + + # Load dependencies + spack load cmake%gcc \$USE_SPECS + spack find --loaded + + rm -rf build/ &> /dev/null + + # Build ANT-MOC + cmake -S . -B build \ + -DCMAKE_C_COMPILER=\$C_COMPILER \ + -DCMAKE_CXX_COMPILER=\$CXX_COMPILER \ + -DCMAKE_BUILD_TYPE=\$BUILD_TYPE \ + -DBUILD_SHARED_LIBS:BOOL=\$BUILD_SHARED_LIBS \ + -DENABLE_TESTS:BOOL=\$ENABLE_TESTS \ + -DENABLE_MPI:BOOL=\$ENABLE_MPI \ + -DENABLE_HIP:BOOL=\$ENABLE_HIP + + echo -e "Building ANT-MOC..." + cmake --build build -j\$(nproc) &> /dev/null + + if [ \$stage == "run" ]; then + ARGS="--output-on-failure" + if [ "\$CTEST_RANDOM" == "ON" ]; then ARGS="\$ARGS --schedule-random"; fi + cd build/ + # Exclude broken tests + # FIXME: these tests are broken on Ubuntu jammy but work on Ubuntu focal + ctest \$ARGS -E unit_test_initialize* -V + fi done + +# Restore permissions +sudo chown -R \$ORIGIN_OWNER tests EOF \ No newline at end of file diff --git a/scripts/docker-slim.sh b/scripts/docker-slim.sh index 981e7a1..9c0787a 100755 --- a/scripts/docker-slim.sh +++ b/scripts/docker-slim.sh @@ -1,8 +1,7 @@ IMAGE=antmoc/antmoc-ci:0.1.16-alpha slim build --target $IMAGE \ - --mount $(pwd)/ant-moc:/home/hpcer/ant-moc \ - --mount $(pwd)/scripts:/home/hpcer/scripts \ + --mount $(pwd)/ant-moc:/opt/mnt/ant-moc \ --http-probe=false \ --show-clogs \ --include-path /opt/spack \ From cd336a8e4d20d5f4c2ebb2f8abe2aea73299342a Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 22:09:46 +0800 Subject: [PATCH 32/73] Update build-antmoc.sh --- scripts/build-antmoc.sh | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) mode change 100755 => 100644 scripts/build-antmoc.sh diff --git a/scripts/build-antmoc.sh b/scripts/build-antmoc.sh old mode 100755 new mode 100644 index 1e14607..05021ba --- a/scripts/build-antmoc.sh +++ b/scripts/build-antmoc.sh @@ -15,13 +15,10 @@ WORKDIR=\$HOME/ant-moc # Always mount ANT-MOC to this directory in containers # -v ./ant-moc:/opt/mnt/ant-moc -cp -r /opt/mnt/ant-moc \$WORKDIR +sudo cp -r /opt/mnt/ant-moc \$WORKDIR +sudo chown -R hpcer:hpcer \$WORKDIR cd \$WORKDIR -# Change permissions -ORIGIN_OWNER=\$(stat -c '%u:%g' tests) -sudo chown -R hpcer:hpcer tests - # Setup environment source \$HOME/setup-env.sh spack debug report @@ -41,7 +38,7 @@ declare -A MPIS=( \ # Test cases declare -a TESTS=( \ - "gcc serial run" "gcc mpich run" "gcc openmpi run" \ + "gcc serial run" "gcc openmpi run" \ "clang serial run" "clang mpich run" \ "hipcc serial build" "hipcc mpich build" "hipcc openmpi build") @@ -104,10 +101,7 @@ for s in "\${TESTS[@]}"; do cd build/ # Exclude broken tests # FIXME: these tests are broken on Ubuntu jammy but work on Ubuntu focal - ctest \$ARGS -E unit_test_initialize* -V + ctest \$ARGS -E unit_test_initialize* fi done - -# Restore permissions -sudo chown -R \$ORIGIN_OWNER tests -EOF \ No newline at end of file +EOF From b9513707a6c4382a9e428ac8db2f879855dd3d7d Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 22:15:20 +0800 Subject: [PATCH 33/73] Add python to spack external, update hdf5 --- Dockerfile | 20 +++++++++++--------- repo/packages/antmoc/package.py | 6 +++--- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7e3b2a8..96fa401 100644 --- a/Dockerfile +++ b/Dockerfile @@ -104,7 +104,8 @@ RUN spack compiler find \ cmake \ gmake \ libtool \ - perl + perl \ + python #------------------------------------------------------------------------------- # Install dependencies for antmoc @@ -117,14 +118,15 @@ ARG CLANG_SPEC="clang" ARG MPICH_SPEC="mpich~fortran" ARG OPENMPI_SPEC="openmpi" -RUN deps=(\ - "cmake %$GCC_SPEC" \ - "lcov@=2.0 %$GCC_SPEC" \ - "antmoc %$CLANG_SPEC ~mpi" \ - "antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC" \ - "antmoc %$GCC_SPEC ~mpi" \ - "antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC" \ - "antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC") \ +RUN set -e; \ + deps=(\ + "cmake %$GCC_SPEC" \ + "lcov@=2.0 %$GCC_SPEC" \ + "antmoc %$CLANG_SPEC ~mpi" \ + "antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC" \ + "antmoc %$GCC_SPEC ~mpi" \ + "antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC" \ + "antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC") \ && for dep in "${deps[@]}"; do spack install -j $(nproc) --fail-fast -ny $dep; done \ && spack gc -y && spack clean -a \ && spack debug report && spack find -v # Check spack and dependency installation diff --git a/repo/packages/antmoc/package.py b/repo/packages/antmoc/package.py index 9898973..dd9a757 100644 --- a/repo/packages/antmoc/package.py +++ b/repo/packages/antmoc/package.py @@ -20,6 +20,6 @@ class Antmoc(BundlePackage): depends_on('fmt@6.0.0:8.0.0+shared') depends_on('tinyxml2@7.0:9.0+shared') depends_on('toml11@3.6:3.7') - depends_on('hdf5@=1.10.8~mpi', when='~mpi') - depends_on('hdf5@=1.10.8+mpi', when='+mpi') - depends_on('googletest@1.10:1.12 +gmock') \ No newline at end of file + depends_on('hdf5@=1.10.11~mpi', when='~mpi') + depends_on('hdf5@=1.10.11+mpi', when='+mpi') + depends_on('googletest@1.10:1.12 +gmock') From 8aa2f22acbeb96836212e2c73d8bd42fd81f96f3 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 23:08:38 +0800 Subject: [PATCH 34/73] Update build-antmoc.sh for testing --- scripts/build-antmoc.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/build-antmoc.sh b/scripts/build-antmoc.sh index 05021ba..edf619f 100644 --- a/scripts/build-antmoc.sh +++ b/scripts/build-antmoc.sh @@ -38,7 +38,7 @@ declare -A MPIS=( \ # Test cases declare -a TESTS=( \ - "gcc serial run" "gcc openmpi run" \ + "gcc openmpi run" \ "clang serial run" "clang mpich run" \ "hipcc serial build" "hipcc mpich build" "hipcc openmpi build") @@ -77,6 +77,7 @@ for s in "\${TESTS[@]}"; do fi # Load dependencies + spack unload --all spack load cmake%gcc \$USE_SPECS spack find --loaded @@ -99,9 +100,12 @@ for s in "\${TESTS[@]}"; do ARGS="--output-on-failure" if [ "\$CTEST_RANDOM" == "ON" ]; then ARGS="\$ARGS --schedule-random"; fi cd build/ - # Exclude broken tests - # FIXME: these tests are broken on Ubuntu jammy but work on Ubuntu focal - ctest \$ARGS -E unit_test_initialize* + # FIXME: some tests are broken on Ubuntu jammy but work on Ubuntu focal + if [ \$mpi == serial ]; then + ctest \$ARGS -E unit_test_initialize* + else + ctest \$ARGS -E unit_mpi_initialize* + fi fi done EOF From 8ddc6f961302074115879daac36be7f4cf2b92e8 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 23:12:25 +0800 Subject: [PATCH 35/73] Update mpi versions --- Dockerfile | 4 ++-- scripts/build-antmoc.sh | 9 +-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 96fa401..51e30be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -115,8 +115,8 @@ ARG GCC_SPEC="gcc" ARG CLANG_SPEC="clang" # MPI specs -ARG MPICH_SPEC="mpich~fortran" -ARG OPENMPI_SPEC="openmpi" +ARG MPICH_SPEC="mpich@=3.4.3" +ARG OPENMPI_SPEC="openmpi@=4.0.7" RUN set -e; \ deps=(\ diff --git a/scripts/build-antmoc.sh b/scripts/build-antmoc.sh index edf619f..5ddc1c4 100644 --- a/scripts/build-antmoc.sh +++ b/scripts/build-antmoc.sh @@ -1,12 +1,5 @@ #!/usr/bin/env bash # For slim toolkit 1.40.11 - -sudo -Hi -u root bash << EOF -rm -f /usr/bin/clang /usr/bin/clang++ -ln -s /usr/bin/clang-14 /usr/bin/clang -ln -s /usr/bin/clang++-14 /usr/bin/clang++ -EOF - sudo -Hi -u hpcer bash << EOF set -e whoami @@ -104,7 +97,7 @@ for s in "\${TESTS[@]}"; do if [ \$mpi == serial ]; then ctest \$ARGS -E unit_test_initialize* else - ctest \$ARGS -E unit_mpi_initialize* + ctest \$ARGS -E unit_mpi_test_initialize* fi fi done From 9fd0b1cdef6090e8be5532c9c6522d186eaee782 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 5 Apr 2024 23:23:06 +0800 Subject: [PATCH 36/73] Roll back mpi versions --- Dockerfile | 4 ++-- scripts/build-antmoc.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 51e30be..5d78ca1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -115,8 +115,8 @@ ARG GCC_SPEC="gcc" ARG CLANG_SPEC="clang" # MPI specs -ARG MPICH_SPEC="mpich@=3.4.3" -ARG OPENMPI_SPEC="openmpi@=4.0.7" +ARG MPICH_SPEC="mpich@=3.3.2~fortran" +ARG OPENMPI_SPEC="openmpi@=4.0.6" RUN set -e; \ deps=(\ diff --git a/scripts/build-antmoc.sh b/scripts/build-antmoc.sh index 5ddc1c4..fede20e 100644 --- a/scripts/build-antmoc.sh +++ b/scripts/build-antmoc.sh @@ -31,9 +31,9 @@ declare -A MPIS=( \ # Test cases declare -a TESTS=( \ - "gcc openmpi run" \ + "gcc serial run" "gcc mpich run" "gcc openmpi run" \ "clang serial run" "clang mpich run" \ - "hipcc serial build" "hipcc mpich build" "hipcc openmpi build") + "hipcc serial build" "hipcc mpich build" "hipcc openmpi build" ) # Run tests for s in "\${TESTS[@]}"; do From 4d7ca1b4484aefaef2269a4a86601fe25ead6e75 Mon Sep 17 00:00:00 2001 From: one Date: Sat, 6 Apr 2024 16:24:23 +0800 Subject: [PATCH 37/73] Update entrypoint and cmd --- Dockerfile | 4 ++-- scripts/docker-slim.sh | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5d78ca1..84bc6a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -163,10 +163,10 @@ RUN (echo "#!/usr/bin/env bash" \ && chmod u+x ~/setup-env.sh #------------------------------------------------------------------------------- -# Reset the entrypoint, add CMD +# Reset the entrypoint and CMD #------------------------------------------------------------------------------- ENTRYPOINT ["/bin/bash"] -CMD ["interactive-shell"] +CMD [] #----------------------------------------------------------------------- # Build-time metadata as defined at http://label-schema.org diff --git a/scripts/docker-slim.sh b/scripts/docker-slim.sh index 9c0787a..a1f6103 100755 --- a/scripts/docker-slim.sh +++ b/scripts/docker-slim.sh @@ -1,11 +1,14 @@ +# See https://github.com/slimtoolkit/slim + IMAGE=antmoc/antmoc-ci:0.1.16-alpha slim build --target $IMAGE \ --mount $(pwd)/ant-moc:/opt/mnt/ant-moc \ --http-probe=false \ --show-clogs \ + --include-shell \ + --include-path /etc \ --include-path /opt/spack \ - --include-path /etc/spack \ --include-path /opt/software \ --include-path /opt/repo \ --entrypoint /bin/bash \ From 707831dc2436d04b4320399f657fe9394ec1c4b2 Mon Sep 17 00:00:00 2001 From: one Date: Sat, 6 Apr 2024 16:26:18 +0800 Subject: [PATCH 38/73] Add a parameter to docker-slim.sh --- scripts/docker-slim.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docker-slim.sh b/scripts/docker-slim.sh index a1f6103..bf029b9 100755 --- a/scripts/docker-slim.sh +++ b/scripts/docker-slim.sh @@ -1,6 +1,6 @@ # See https://github.com/slimtoolkit/slim -IMAGE=antmoc/antmoc-ci:0.1.16-alpha +IMAGE={1:-"antmoc/antmoc-ci:0.1.16-alpha"} slim build --target $IMAGE \ --mount $(pwd)/ant-moc:/opt/mnt/ant-moc \ From 808bd970ea48cf24f7e614d9e9b79b7c56e4f090 Mon Sep 17 00:00:00 2001 From: one Date: Sat, 6 Apr 2024 16:27:17 +0800 Subject: [PATCH 39/73] Fix docker-slim.sh parameters --- scripts/docker-slim.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docker-slim.sh b/scripts/docker-slim.sh index bf029b9..8925ec6 100755 --- a/scripts/docker-slim.sh +++ b/scripts/docker-slim.sh @@ -1,6 +1,6 @@ # See https://github.com/slimtoolkit/slim -IMAGE={1:-"antmoc/antmoc-ci:0.1.16-alpha"} +IMAGE=${1:-"antmoc/antmoc-ci:0.1.16-alpha"} slim build --target $IMAGE \ --mount $(pwd)/ant-moc:/opt/mnt/ant-moc \ From 4e190f4b702cf458ed8f350aa699a4c569b60a00 Mon Sep 17 00:00:00 2001 From: one Date: Sat, 6 Apr 2024 16:45:58 +0800 Subject: [PATCH 40/73] Update MPI versions --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 84bc6a3..20b8028 100644 --- a/Dockerfile +++ b/Dockerfile @@ -115,8 +115,8 @@ ARG GCC_SPEC="gcc" ARG CLANG_SPEC="clang" # MPI specs -ARG MPICH_SPEC="mpich@=3.3.2~fortran" -ARG OPENMPI_SPEC="openmpi@=4.0.6" +ARG MPICH_SPEC="mpich@=3.4.3" +ARG OPENMPI_SPEC="openmpi@=4.0.7" RUN set -e; \ deps=(\ From 4c6fc5d4a66525927284098097dd9f4d12eb1d42 Mon Sep 17 00:00:00 2001 From: one Date: Sat, 6 Apr 2024 17:34:12 +0800 Subject: [PATCH 41/73] Remove setup-env.sh --- Dockerfile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 20b8028..1552f65 100644 --- a/Dockerfile +++ b/Dockerfile @@ -115,7 +115,7 @@ ARG GCC_SPEC="gcc" ARG CLANG_SPEC="clang" # MPI specs -ARG MPICH_SPEC="mpich@=3.4.3" +ARG MPICH_SPEC="mpich@=3.4.3~fortran" ARG OPENMPI_SPEC="openmpi@=4.0.7" RUN set -e; \ @@ -151,21 +151,19 @@ USER $USER_NAME WORKDIR /home/$USER_NAME # generate a script for Spack -RUN (echo "#!/usr/bin/env bash" \ +RUN (echo "export SPACK_ROOT=$SPACK_ROOT" \ +&& echo ". $SPACK_ROOT/share/spack/setup-env.sh" \ # && echo "export PATH=\$PATH:/opt/rocm/bin:/opt/rocm/rocprofiler/bin:/opt/rocm/opencl/bin" \ # && echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/opt/rocm/lib:/opt/rocm/hip/lib:/opt/rocm/llvm/lib:/opt/rocm/opencl/lib" \ # && echo "export INCLUDE=\$INCLUDE:/opt/rocm/include:/opt/rocm/hip/include:/opt/rocm/llvm/include" \ # && echo "export C_INCLUDE_PATH=\$C_INCLUDE_PATH:/opt/rocm/include:/opt/rocm/hip/include:/opt/rocm/llvm/include" \ # && echo "export CPLUS_INCLUDE_PATH=\$CPLUS_INCLUDE_PATH:/opt/rocm/include:/opt/rocm/hip/include:/opt/rocm/llvm/include" \ -&& echo "export SPACK_ROOT=$SPACK_ROOT" \ -&& echo ". $SPACK_ROOT/share/spack/setup-env.sh" \ -&& echo "") > ~/setup-env.sh \ -&& chmod u+x ~/setup-env.sh +&& echo "") > ~/.bashrc #------------------------------------------------------------------------------- # Reset the entrypoint and CMD #------------------------------------------------------------------------------- -ENTRYPOINT ["/bin/bash"] +ENTRYPOINT ["/bin/bash", "-l"] CMD [] #----------------------------------------------------------------------- From 271bd680584cf6147dcca0b907cc39fd8107a233 Mon Sep 17 00:00:00 2001 From: one Date: Sat, 6 Apr 2024 17:34:35 +0800 Subject: [PATCH 42/73] Update scripts --- scripts/build-antmoc.sh | 14 ++++++-------- scripts/docker-slim.sh | 5 ++++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/scripts/build-antmoc.sh b/scripts/build-antmoc.sh index fede20e..19bfa9f 100644 --- a/scripts/build-antmoc.sh +++ b/scripts/build-antmoc.sh @@ -15,7 +15,7 @@ cd \$WORKDIR # Setup environment source \$HOME/setup-env.sh spack debug report -spack find -v +spack find -v antmoc # Compilers in tuple (C compiler, C++ compiler, spack spec) declare -A COMPILERS=( \ @@ -87,18 +87,16 @@ for s in "\${TESTS[@]}"; do -DENABLE_HIP:BOOL=\$ENABLE_HIP echo -e "Building ANT-MOC..." - cmake --build build -j\$(nproc) &> /dev/null + cmake --build build -j\$(nproc) if [ \$stage == "run" ]; then ARGS="--output-on-failure" if [ "\$CTEST_RANDOM" == "ON" ]; then ARGS="\$ARGS --schedule-random"; fi cd build/ - # FIXME: some tests are broken on Ubuntu jammy but work on Ubuntu focal - if [ \$mpi == serial ]; then - ctest \$ARGS -E unit_test_initialize* - else - ctest \$ARGS -E unit_mpi_test_initialize* - fi + ctest \$ARGS fi done + +cd +rm -rf \$WORKDIR EOF diff --git a/scripts/docker-slim.sh b/scripts/docker-slim.sh index 8925ec6..d55f89f 100755 --- a/scripts/docker-slim.sh +++ b/scripts/docker-slim.sh @@ -2,11 +2,14 @@ IMAGE=${1:-"antmoc/antmoc-ci:0.1.16-alpha"} +# /opt is the largest directory slim build --target $IMAGE \ --mount $(pwd)/ant-moc:/opt/mnt/ant-moc \ --http-probe=false \ --show-clogs \ - --include-shell \ + --include-path /root \ + --include-path /home \ + --include-path /usr \ --include-path /etc \ --include-path /opt/spack \ --include-path /opt/software \ From e91b2b131ed515c204338d914eeb7c593889c859 Mon Sep 17 00:00:00 2001 From: one Date: Sat, 6 Apr 2024 17:39:36 +0800 Subject: [PATCH 43/73] SImplify spack specs in Dockerfile --- Dockerfile | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1552f65..463308b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -110,23 +110,19 @@ RUN spack compiler find \ #------------------------------------------------------------------------------- # Install dependencies for antmoc #------------------------------------------------------------------------------- -# Compiler specs -ARG GCC_SPEC="gcc" -ARG CLANG_SPEC="clang" - # MPI specs ARG MPICH_SPEC="mpich@=3.4.3~fortran" ARG OPENMPI_SPEC="openmpi@=4.0.7" RUN set -e; \ deps=(\ - "cmake %$GCC_SPEC" \ - "lcov@=2.0 %$GCC_SPEC" \ - "antmoc %$CLANG_SPEC ~mpi" \ - "antmoc %$CLANG_SPEC +mpi ^$MPICH_SPEC" \ - "antmoc %$GCC_SPEC ~mpi" \ - "antmoc %$GCC_SPEC +mpi ^$MPICH_SPEC" \ - "antmoc %$GCC_SPEC +mpi ^$OPENMPI_SPEC") \ + "cmake %gcc" \ + "lcov@=2.0 %gcc" \ + "antmoc %clang ~mpi" \ + "antmoc %clang +mpi ^$MPICH_SPEC %clang" \ + "antmoc %gcc ~mpi" \ + "antmoc %gcc +mpi ^$MPICH_SPEC" \ + "antmoc %gcc +mpi ^$OPENMPI_SPEC") \ && for dep in "${deps[@]}"; do spack install -j $(nproc) --fail-fast -ny $dep; done \ && spack gc -y && spack clean -a \ && spack debug report && spack find -v # Check spack and dependency installation From 56495e719a63f6608b0ce8805ffb033bb48311b5 Mon Sep 17 00:00:00 2001 From: one Date: Sat, 6 Apr 2024 22:59:49 +0800 Subject: [PATCH 44/73] Update spack package versions --- repo/packages/antmoc/package.py | 10 +-- repo/packages/googletest/package.py | 95 +++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 repo/packages/googletest/package.py diff --git a/repo/packages/antmoc/package.py b/repo/packages/antmoc/package.py index dd9a757..12c72d1 100644 --- a/repo/packages/antmoc/package.py +++ b/repo/packages/antmoc/package.py @@ -16,10 +16,10 @@ class Antmoc(BundlePackage): depends_on('cmake@3.16:', type='build') depends_on('mpi@=3.0', when='+mpi', type=('build', 'link', 'run')) - depends_on('cxxopts@=3.0.0') + depends_on('cxxopts@3') depends_on('fmt@6.0.0:8.0.0+shared') - depends_on('tinyxml2@7.0:9.0+shared') + depends_on('tinyxml2@7.0:10.0+shared') depends_on('toml11@3.6:3.7') - depends_on('hdf5@=1.10.11~mpi', when='~mpi') - depends_on('hdf5@=1.10.11+mpi', when='+mpi') - depends_on('googletest@1.10:1.12 +gmock') + depends_on('hdf5@1.12~mpi', when='~mpi') + depends_on('hdf5@1.12+mpi', when='+mpi') + depends_on('googletest@1.14 +gmock') diff --git a/repo/packages/googletest/package.py b/repo/packages/googletest/package.py new file mode 100644 index 0000000..f1b1424 --- /dev/null +++ b/repo/packages/googletest/package.py @@ -0,0 +1,95 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * + + +class Googletest(CMakePackage): + """Google test framework for C++. Also called gtest.""" + + homepage = "https://github.com/google/googletest" + url = "https://github.com/google/googletest/archive/refs/tags/v1.14.0.tar.gz" + git = "https://github.com/google/googletest" + + maintainers("sethrj") + + version("main", branch="main") + version("1.14.0", sha256="8ad598c73ad796e0d8280b082cebd82a630d73e73cd3c70057938a6501bba5d7") + version("1.13.0", sha256="ad7fdba11ea011c1d925b3289cf4af2c66a352e18d4c7264392fead75e919363") + version("1.12.1", sha256="81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2") + version("1.12.0", sha256="2a4f11dce6188b256f3650061525d0fe352069e5c162452818efbbf8d0b5fe1c") + version("1.11.0", sha256="b4870bf121ff7795ba20d20bcdd8627b8e088f2d1dab299a031c1034eddc93d5") + version("1.10.0", sha256="9dc9157a9a1551ec7a7e43daea9a694a0bb5fb8bec81235d8a1e6ef64c716dcb") + version("1.8.1", sha256="9bf1fe5182a604b4135edc1a425ae356c9ad15e9b23f9f12a02e80184c3a249c") + version("1.8.0", sha256="58a6f4277ca2bc8565222b3bbd58a177609e9c488e8a72649359ba51450db7d8") + version("1.7.0", sha256="f73a6546fdf9fce9ff93a5015e0333a8af3062a152a9ad6bcb772c96687016cc") + version("1.6.0", sha256="5fbc058e5b662b9c86d93ac76fefb58eec89cbf26144b49669a38ecb62758447") + + variant("gmock", default=True, when="@1.8:", description="Build with gmock") + variant("pthreads", default=True, description="Build multithreaded version with pthreads") + variant("shared", default=True, description="Build shared libraries (DLLs)") + + variant( + "cxxstd", + default="11", + values=("98", "11", "14", "17"), + multi=False, + description="Use the specified C++ standard when building", + ) + conflicts("cxxstd=98", when="@1.9:") + conflicts("cxxstd=11", when="@1.13:") + + def cmake_args(self): + spec = self.spec + args = [ + self.define_from_variant("gtest_disable_pthreads", "pthreads"), + self.define_from_variant("BUILD_SHARED_LIBS", "shared"), + self.define_from_variant("CMAKE_CXX_STANDARD", "cxxstd"), + ] + if spec.satisfies("@1.8:"): + # New style (contains both Google Mock and Google Test) + args.append(self.define("BUILD_GTEST", True)) + args.append(self.define_from_variant("BUILD_GMOCK", "gmock")) + + return args + + @when("@:1.7.0") + def install(self, spec, prefix): + """Make the install targets""" + with working_dir(self.build_directory): + # Google Test doesn't have a make install + # We have to do our own install here. + install_tree(join_path(self.stage.source_path, "include"), prefix.include) + + mkdirp(prefix.lib) + if "+shared" in spec: + install("libgtest.{0}".format(dso_suffix), prefix.lib) + install("libgtest_main.{0}".format(dso_suffix), prefix.lib) + else: + install("libgtest.a", prefix.lib) + install("libgtest_main.a", prefix.lib) + + @run_after("install") + def darwin_fix(self): + # The shared library is not installed correctly on Darwin; fix this + if self.spec.satisfies("platform=darwin"): + fix_darwin_install_name(self.prefix.lib) + + def url_for_version(self, version): + """googletest has changed how they publish releases on github. Up until, + including version 1.12.1 they were tagged as `release-`. + Afterwards things switched to the format `v`. Additionally, + newer versions are available from `archive/refs/tags/.tar.gz`, + while versions up to, and including, 1.8.0 are available only from + `archive/release-.tar.gz` + """ + if version <= Version("1.8.0"): + return f"{self.git}/archive/release-{version}.tar.gz" + + tagname = f"release-{version}" + if version >= Version("1.13"): + tagname = f"v{version}" + + return f"{self.git}/archive/refs/tags/{tagname}.tar.gz" From 715b6bf29b4016f417ffade79506ebbaf0b0580e Mon Sep 17 00:00:00 2001 From: one Date: Sat, 6 Apr 2024 23:09:59 +0800 Subject: [PATCH 45/73] Update spack specs --- Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 463308b..1773e19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -104,6 +104,7 @@ RUN spack compiler find \ cmake \ gmake \ libtool \ + openssh \ perl \ python @@ -112,8 +113,10 @@ RUN spack compiler find \ #------------------------------------------------------------------------------- # MPI specs ARG MPICH_SPEC="mpich@=3.4.3~fortran" -ARG OPENMPI_SPEC="openmpi@=4.0.7" +ARG OPENMPI_SPEC="openmpi@4.1" +# To avoid the default --reuse option of spack 0.21, +# add %clang and %gcc for every MPI spec. RUN set -e; \ deps=(\ "cmake %gcc" \ @@ -121,8 +124,8 @@ RUN set -e; \ "antmoc %clang ~mpi" \ "antmoc %clang +mpi ^$MPICH_SPEC %clang" \ "antmoc %gcc ~mpi" \ - "antmoc %gcc +mpi ^$MPICH_SPEC" \ - "antmoc %gcc +mpi ^$OPENMPI_SPEC") \ + "antmoc %gcc +mpi ^$MPICH_SPEC %gcc" \ + "antmoc %gcc +mpi ^$OPENMPI_SPEC %gcc") \ && for dep in "${deps[@]}"; do spack install -j $(nproc) --fail-fast -ny $dep; done \ && spack gc -y && spack clean -a \ && spack debug report && spack find -v # Check spack and dependency installation From 8423a7ee79b9c597f82fd7e526a570e08d8a8c85 Mon Sep 17 00:00:00 2001 From: one Date: Sat, 6 Apr 2024 23:20:21 +0800 Subject: [PATCH 46/73] Fix build-antmoc.sh --- scripts/build-antmoc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build-antmoc.sh b/scripts/build-antmoc.sh index 19bfa9f..41b44e8 100644 --- a/scripts/build-antmoc.sh +++ b/scripts/build-antmoc.sh @@ -4,7 +4,7 @@ sudo -Hi -u hpcer bash << EOF set -e whoami -WORKDIR=\$HOME/ant-moc +WORKDIR=/tmp/ant-moc # Always mount ANT-MOC to this directory in containers # -v ./ant-moc:/opt/mnt/ant-moc @@ -13,7 +13,7 @@ sudo chown -R hpcer:hpcer \$WORKDIR cd \$WORKDIR # Setup environment -source \$HOME/setup-env.sh +source \$HOME/.bashrc spack debug report spack find -v antmoc From bf4ecc36cfa8906691f609e881909fd166b73a66 Mon Sep 17 00:00:00 2001 From: one Date: Sun, 7 Apr 2024 02:23:41 +0800 Subject: [PATCH 47/73] Use heredoc, use AMD clang instead of the clang shipped with Ubuntu --- Dockerfile | 151 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 95 insertions(+), 56 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1773e19..0621849 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,18 +37,18 @@ RUN sed -i -e "s/jammy/$UBUNTU_CODE/g" /etc/apt/sources.list \ rocm-dev \ rocthrust-dev \ build-essential -# install LLVM and CMake for spack -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - llvm-14 \ - clang-14 \ - libomp-14-dev \ - cmake \ - openssh-server \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && rm -f /usr/bin/clang /usr/bin/clang++ &> /dev/null \ - && ln -s /usr/bin/clang-14 /usr/bin/clang \ - && ln -s /usr/bin/clang++-14 /usr/bin/clang++ +# set up AMD clang and install CMake for spack +RUN < /dev/null +ln -s /opt/rocm-${ROCM_VERSION}/llvm/bin/clang /usr/bin/clang +ln -s /opt/rocm-${ROCM_VERSION}/llvm/bin/clang++ /usr/bin/clang++ +EOF #------------------------------------------------------------------------------- # Set up spack @@ -61,11 +61,13 @@ ENV MIRROR_DIR=/opt/mirror ENV REPO_DIR=/opt/repo # create directories for Spack -RUN set -e; \ - mkdir -p $CONFIG_DIR; \ - mkdir -p $INSTALL_DIR; \ - mkdir -p $REPO_DIR; \ - mkdir -p $MIRROR_DIR +RUN < $CONFIG_DIR/config.yaml \ - && (echo "mirrors:" \ - && echo " local: file://$MIRROR_DIR") > $CONFIG_DIR/mirrors.yaml \ - && (echo "repos:" \ - && echo " - $REPO_DIR") > $CONFIG_DIR/repos.yaml \ - && (echo "packages:" \ - && echo " all:" \ - && echo " target: [$TARGET]") > $CONFIG_DIR/packages.yaml +COPY < $CONFIG_DIR/compilers.yaml \ - && spack compiler list \ - && spack external find --scope system --not-buildable \ - gcc \ - llvm \ +# manually add AMD clang to compilers, 'CLANG_VERSION' is a placeholder +COPY < /dev/null; then \ - useradd -m $USER_NAME; \ - echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers; \ - fi +RUN < /dev/null; then + useradd -m $USER_NAME + echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +fi +EOF # transfer control to the default user USER $USER_NAME From 1009588fb123230f83b4fb686fa4155f91c29f8f Mon Sep 17 00:00:00 2001 From: one Date: Sun, 7 Apr 2024 02:40:04 +0800 Subject: [PATCH 48/73] Use spack to get ubuntu version --- Dockerfile | 8 +++++--- scripts/build-antmoc.sh | 8 +++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0621849..8634b89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -113,8 +113,8 @@ compilers: f77: /usr/bin/gfortran fc: /usr/bin/gfortran flags: {} - operating_system: ubuntu22.04 - target: x86_64 + operating_system: UBUNTU_VERSION + target: $TARGET modules: [] environment: {} extra_rpaths: [] @@ -123,8 +123,10 @@ EOF # find gcc and external packages RUN < Date: Sun, 7 Apr 2024 09:57:48 +0800 Subject: [PATCH 49/73] Minor fix --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8634b89..8b0f597 100644 --- a/Dockerfile +++ b/Dockerfile @@ -129,7 +129,7 @@ sed -i -e "s/UBUNTU_VERSION/$(spack debug report | grep -Po '(?<=linux-)[^-]+')/ # find gcc spack compiler find --scope system -spack compiler list +spack config get compilers > $CONFIG_DIR/compilers.yaml # find external packages spack external find --scope system --not-buildable \ autoconf \ @@ -198,7 +198,7 @@ RUN (echo "export SPACK_ROOT=$SPACK_ROOT" \ # && echo "export INCLUDE=\$INCLUDE:/opt/rocm/include:/opt/rocm/hip/include:/opt/rocm/llvm/include" \ # && echo "export C_INCLUDE_PATH=\$C_INCLUDE_PATH:/opt/rocm/include:/opt/rocm/hip/include:/opt/rocm/llvm/include" \ # && echo "export CPLUS_INCLUDE_PATH=\$CPLUS_INCLUDE_PATH:/opt/rocm/include:/opt/rocm/hip/include:/opt/rocm/llvm/include" \ -&& echo "") > ~/.bashrc +&& echo "") >> ~/.bashrc #------------------------------------------------------------------------------- # Reset the entrypoint and CMD From 6f5900af658306b66dcafd666fb427fac0c1a4cc Mon Sep 17 00:00:00 2001 From: one Date: Sun, 7 Apr 2024 14:05:18 +0800 Subject: [PATCH 50/73] Fix missing compilers in the system spack config file --- Dockerfile | 113 +++++++++++++++++++++++++++-------------------------- 1 file changed, 58 insertions(+), 55 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8b0f597..53b804a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,29 +17,36 @@ ARG ROCM_VERSION=5.4.6 ARG AMDGPU_VERSION=5.4.6 # install ROCm HIP, see https://github.com/ROCm/ROCm-docker/blob/master/dev/Dockerfile-ubuntu-22.04 -COPY etc/apt/ /etc/apt/ +ADD etc/apt/ /etc/apt/ ARG APT_PREF="Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600" -RUN sed -i -e "s/jammy/$UBUNTU_CODE/g" /etc/apt/sources.list \ - && echo -e "$APT_PREF" | tee /etc/apt/preferences.d/rocm-pin-600 \ - && apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl libnuma-dev gnupg \ - && curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - \ - && printf "deb [arch=amd64] https://repo.radeon.com/rocm/apt/$ROCM_VERSION/ $UBUNTU_CODE main" | tee /etc/apt/sources.list.d/rocm.list \ - && printf "deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu $UBUNTU_CODE main" | tee /etc/apt/sources.list.d/amdgpu.list \ - && apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - sudo \ - libelf1 \ - kmod \ - file \ - python3 \ - python3-pip \ - rocm-dev \ - rocthrust-dev \ - build-essential -# set up AMD clang and install CMake for spack -RUN < $CONFIG_DIR/config.yaml config: install_tree: root: $INSTALL_DIR connect_timeout: 600 -EOF +EOF1 -COPY < $CONFIG_DIR/packages.yaml packages: all: target: [$TARGET] -EOF +EOF2 -RUN < $CONFIG_DIR/compilers.yaml compilers: - compiler: spec: clang@=CLANG_VERSION @@ -118,20 +126,15 @@ compilers: modules: [] environment: {} extra_rpaths: [] -EOF +EOF1 -# find gcc and external packages -RUN < $CONFIG_DIR/compilers.yaml # find external packages spack external find --scope system --not-buildable \ + gcc \ autoconf \ automake \ cmake \ @@ -152,8 +155,8 @@ ARG OPENMPI_SPEC="openmpi@=4.1.6" # To avoid the default --reuse option of spack 0.21, # add %clang and %gcc for every MPI spec. -RUN < /dev/null; then - useradd -m $USER_NAME + useradd -m $USER_NAME -d $USER_HOME echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers fi +# copy spack configuration of the root to the new user's home +cp -r ~/.spack $USER_HOME/ +# change permissions (could be time-consuming) +chown -R $USER_NAME: $USER_HOME/.spack $CONFIG_DIR $REPO_DIR $MIRROR_DIR $INSTALL_DIR EOF # transfer control to the default user USER $USER_NAME -WORKDIR /home/$USER_NAME +WORKDIR $USER_HOME # generate a script for Spack RUN (echo "export SPACK_ROOT=$SPACK_ROOT" \ && echo ". $SPACK_ROOT/share/spack/setup-env.sh" \ -# && echo "export PATH=\$PATH:/opt/rocm/bin:/opt/rocm/rocprofiler/bin:/opt/rocm/opencl/bin" \ -# && echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/opt/rocm/lib:/opt/rocm/hip/lib:/opt/rocm/llvm/lib:/opt/rocm/opencl/lib" \ -# && echo "export INCLUDE=\$INCLUDE:/opt/rocm/include:/opt/rocm/hip/include:/opt/rocm/llvm/include" \ -# && echo "export C_INCLUDE_PATH=\$C_INCLUDE_PATH:/opt/rocm/include:/opt/rocm/hip/include:/opt/rocm/llvm/include" \ -# && echo "export CPLUS_INCLUDE_PATH=\$CPLUS_INCLUDE_PATH:/opt/rocm/include:/opt/rocm/hip/include:/opt/rocm/llvm/include" \ && echo "") >> ~/.bashrc #------------------------------------------------------------------------------- From 822e19a59db8f8e5a0bd006e539ffa20d1cb0ad7 Mon Sep 17 00:00:00 2001 From: one Date: Tue, 9 Apr 2024 01:41:52 +0800 Subject: [PATCH 51/73] Update dependency versions --- repo/packages/antmoc/package.py | 10 +++++----- scripts/build-antmoc.sh | 7 +++++-- 2 files changed, 10 insertions(+), 7 deletions(-) mode change 100644 => 100755 scripts/build-antmoc.sh diff --git a/repo/packages/antmoc/package.py b/repo/packages/antmoc/package.py index 12c72d1..dbdb6a5 100644 --- a/repo/packages/antmoc/package.py +++ b/repo/packages/antmoc/package.py @@ -17,9 +17,9 @@ class Antmoc(BundlePackage): depends_on('cmake@3.16:', type='build') depends_on('mpi@=3.0', when='+mpi', type=('build', 'link', 'run')) depends_on('cxxopts@3') - depends_on('fmt@6.0.0:8.0.0+shared') - depends_on('tinyxml2@7.0:10.0+shared') + depends_on('fmt@8.0:9.0 +shared') + depends_on('tinyxml2@7.0:10.0 +shared') depends_on('toml11@3.6:3.7') - depends_on('hdf5@1.12~mpi', when='~mpi') - depends_on('hdf5@1.12+mpi', when='+mpi') - depends_on('googletest@1.14 +gmock') + depends_on('hdf5@1.12:1.14 ~mpi', when='~mpi') + depends_on('hdf5@1.12:1.14 +mpi', when='+mpi') + depends_on('googletest@1.10.0: +gmock') diff --git a/scripts/build-antmoc.sh b/scripts/build-antmoc.sh old mode 100644 new mode 100755 index 45b7ace..af7c75d --- a/scripts/build-antmoc.sh +++ b/scripts/build-antmoc.sh @@ -5,11 +5,12 @@ set -e whoami WORKDIR=/tmp/ant-moc +[ -d \$WORKDIR ] && rm -rf \$WORKDIR +mkdir \$WORKDIR # Always mount ANT-MOC to this directory in containers # -v ./ant-moc:/opt/mnt/ant-moc -sudo cp -r /opt/mnt/ant-moc \$WORKDIR -sudo chown -R hpcer:hpcer \$WORKDIR +cp -r /opt/mnt/ant-moc/. \$WORKDIR/ cd \$WORKDIR # Setup environment @@ -51,6 +52,7 @@ for s in "\${TESTS[@]}"; do USE_SPECS="antmoc \${cc[2]} \${MPIS[\$mpi]}" BUILD_TYPE=Release BUILD_SHARED_LIBS=ON + ENABLE_ALL_WARNINGS=OFF # Enable tests if [ \$stage == "run" ]; then @@ -88,6 +90,7 @@ for s in "\${TESTS[@]}"; do -DCMAKE_CXX_COMPILER=\$CXX_COMPILER \ -DCMAKE_BUILD_TYPE=\$BUILD_TYPE \ -DBUILD_SHARED_LIBS:BOOL=\$BUILD_SHARED_LIBS \ + -DENABLE_ALL_WARNINGS=\$ENABLE_ALL_WARNINGS \ -DENABLE_TESTS:BOOL=\$ENABLE_TESTS \ -DENABLE_MPI:BOOL=\$ENABLE_MPI \ -DENABLE_HIP:BOOL=\$ENABLE_HIP From a9e4ab2c385d42f978ad06a1bb6a44d93b659a74 Mon Sep 17 00:00:00 2001 From: one Date: Tue, 9 Apr 2024 11:16:19 +0800 Subject: [PATCH 52/73] Fix googletest variant pthreads --- repo/packages/googletest/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/repo/packages/googletest/package.py b/repo/packages/googletest/package.py index f1b1424..8b88c9a 100644 --- a/repo/packages/googletest/package.py +++ b/repo/packages/googletest/package.py @@ -44,10 +44,12 @@ class Googletest(CMakePackage): def cmake_args(self): spec = self.spec args = [ - self.define_from_variant("gtest_disable_pthreads", "pthreads"), + # self.define_from_variant("gtest_disable_pthreads", "pthreads"), self.define_from_variant("BUILD_SHARED_LIBS", "shared"), self.define_from_variant("CMAKE_CXX_STANDARD", "cxxstd"), ] + if spec.satisfies("~pthreads"): + args.append(self.define("gtest_disable_pthreads", True)) if spec.satisfies("@1.8:"): # New style (contains both Google Mock and Google Test) args.append(self.define("BUILD_GTEST", True)) From d669bf0ac36216dd7eef05d9dc855e9435df9b8f Mon Sep 17 00:00:00 2001 From: one Date: Tue, 9 Apr 2024 11:45:12 +0800 Subject: [PATCH 53/73] Update package --- repo/packages/antmoc/package.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/repo/packages/antmoc/package.py b/repo/packages/antmoc/package.py index dbdb6a5..a084951 100644 --- a/repo/packages/antmoc/package.py +++ b/repo/packages/antmoc/package.py @@ -4,7 +4,7 @@ class Antmoc(BundlePackage): """This bundle package is only for convenience.""" - homepage = "" + homepage = "https://gitlab.com/HPCer/neutronics/ant-moc" maintainers = ['alephpiece'] @@ -20,6 +20,6 @@ class Antmoc(BundlePackage): depends_on('fmt@8.0:9.0 +shared') depends_on('tinyxml2@7.0:10.0 +shared') depends_on('toml11@3.6:3.7') - depends_on('hdf5@1.12:1.14 ~mpi', when='~mpi') - depends_on('hdf5@1.12:1.14 +mpi', when='+mpi') - depends_on('googletest@1.10.0: +gmock') + depends_on('hdf5@1.12:1.14 ~mpi+shared', when='~mpi') + depends_on('hdf5@1.12:1.14 +mpi+shared', when='+mpi') + depends_on('googletest@1.10.0: +gmock+pthreads+shared') \ No newline at end of file From ca0a849b14bfa2f555333d27bc7a3a63041c9124 Mon Sep 17 00:00:00 2001 From: one Date: Tue, 9 Apr 2024 17:39:45 +0800 Subject: [PATCH 54/73] Update dependencies --- repo/packages/antmoc/package.py | 6 +++--- scripts/{build-antmoc.sh => test-antmoc.sh} | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) rename scripts/{build-antmoc.sh => test-antmoc.sh} (97%) diff --git a/repo/packages/antmoc/package.py b/repo/packages/antmoc/package.py index a084951..a09bfc5 100644 --- a/repo/packages/antmoc/package.py +++ b/repo/packages/antmoc/package.py @@ -17,9 +17,9 @@ class Antmoc(BundlePackage): depends_on('cmake@3.16:', type='build') depends_on('mpi@=3.0', when='+mpi', type=('build', 'link', 'run')) depends_on('cxxopts@3') - depends_on('fmt@8.0:9.0 +shared') - depends_on('tinyxml2@7.0:10.0 +shared') + depends_on('fmt@8:10 +shared') + depends_on('tinyxml2@7:10 +shared') depends_on('toml11@3.6:3.7') depends_on('hdf5@1.12:1.14 ~mpi+shared', when='~mpi') depends_on('hdf5@1.12:1.14 +mpi+shared', when='+mpi') - depends_on('googletest@1.10.0: +gmock+pthreads+shared') \ No newline at end of file + depends_on('googletest@1.10.0: +gmock+pthreads+shared') diff --git a/scripts/build-antmoc.sh b/scripts/test-antmoc.sh similarity index 97% rename from scripts/build-antmoc.sh rename to scripts/test-antmoc.sh index af7c75d..4dcaade 100755 --- a/scripts/build-antmoc.sh +++ b/scripts/test-antmoc.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # For slim toolkit 1.40.11 -sudo -Hi -u hpcer bash << EOF +sudo -Hi -u hpcer bash -i << EOF set -e whoami @@ -14,7 +14,7 @@ cp -r /opt/mnt/ant-moc/. \$WORKDIR/ cd \$WORKDIR # Setup environment -source \$HOME/.bashrc +source ~/.bashrc spack debug report spack find -v antmoc From f70e74c21911673f5e84a0d660288ff7b09c19e0 Mon Sep 17 00:00:00 2001 From: one Date: Tue, 9 Apr 2024 21:48:02 +0800 Subject: [PATCH 55/73] Add a simplified dockerfile without rocm --- Dockerfile | 38 ++----- Dockerfile-full | 223 +++++++++++++++++++++++++++++++++++++++++ Makefile | 30 +++++- scripts/docker-slim.sh | 5 +- 4 files changed, 260 insertions(+), 36 deletions(-) create mode 100644 Dockerfile-full diff --git a/Dockerfile b/Dockerfile index 53b804a..7b39da8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,49 +12,27 @@ WORKDIR /tmp #------------------------------------------------------------------------------- # Install system compilers and libraries #------------------------------------------------------------------------------- -# Register the ROCM package repository, and install rocm-dev package -ARG ROCM_VERSION=5.4.6 -ARG AMDGPU_VERSION=5.4.6 - -# install ROCm HIP, see https://github.com/ROCm/ROCm-docker/blob/master/dev/Dockerfile-ubuntu-22.04 ADD etc/apt/ /etc/apt/ -ARG APT_PREF="Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600" -RUN < /dev/null -ln -s /opt/rocm-${ROCM_VERSION}/llvm/bin/clang /usr/bin/clang -ln -s /opt/rocm-${ROCM_VERSION}/llvm/bin/clang++ /usr/bin/clang++ +ln -s /usr/bin/clang-15 /usr/bin/clang +ln -s /usr/bin/clang++-15 /usr/bin/clang++ EOF #------------------------------------------------------------------------------- @@ -159,7 +137,7 @@ RUN < /dev/null +ln -s /opt/rocm-${ROCM_VERSION}/llvm/bin/clang /usr/bin/clang +ln -s /opt/rocm-${ROCM_VERSION}/llvm/bin/clang++ /usr/bin/clang++ +EOF + +#------------------------------------------------------------------------------- +# Set up spack +#------------------------------------------------------------------------------- +# set Spack paths which should be shared between docker stages +ENV SPACK_ROOT=/opt/spack +ENV CONFIG_DIR=/etc/spack +ENV INSTALL_DIR=/opt/software +ENV MIRROR_DIR=/opt/mirror +ENV REPO_DIR=/opt/repo + +RUN < $CONFIG_DIR/config.yaml +config: + install_tree: + root: $INSTALL_DIR + connect_timeout: 600 +EOF1 + +cat << EOF2 > $CONFIG_DIR/packages.yaml +packages: + all: + target: [$TARGET] +EOF2 + +spack mirror add --scope system local $MIRROR_DIR +spack repo add --scope system $REPO_DIR +EOF + +#------------------------------------------------------------------------------- +# Find system compilers +#------------------------------------------------------------------------------- +RUN < $CONFIG_DIR/compilers.yaml +compilers: +- compiler: + spec: clang@=CLANG_VERSION + paths: + cc: /usr/bin/clang + cxx: /usr/bin/clang++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: UBUNTU_VERSION + target: $TARGET + modules: [] + environment: {} + extra_rpaths: [] +EOF1 + +# substitute clang version and ubuntu version with the correct ones +sed -i -e "s/CLANG_VERSION/$(clang --version | grep -Po '(?<=version )[^ ]+')/g" $CONFIG_DIR/compilers.yaml +sed -i -e "s/UBUNTU_VERSION/$(spack debug report | grep -Po '(?<=linux-)[^-]+')/g" $CONFIG_DIR/compilers.yaml + +# find external packages +spack external find --scope system --not-buildable \ + gcc \ + autoconf \ + automake \ + cmake \ + gmake \ + libtool \ + openssh \ + perl \ + python +EOF + + +#------------------------------------------------------------------------------- +# Install dependencies for antmoc +#------------------------------------------------------------------------------- +# MPI specs +ARG MPICH_SPEC="mpich@=4.1.2~fortran" +ARG OPENMPI_SPEC="openmpi@=4.1.6" + +# To avoid the default --reuse option of spack 0.21, +# add %clang and %gcc for every MPI spec. +RUN < /dev/null; then + useradd -m $USER_NAME -d $USER_HOME + echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +fi +# copy spack configuration of the root to the new user's home +cp -r ~/.spack $USER_HOME/ +# change permissions (could be time-consuming) +chown -R $USER_NAME: $USER_HOME/.spack $CONFIG_DIR $REPO_DIR $MIRROR_DIR $INSTALL_DIR +EOF + +# transfer control to the default user +USER $USER_NAME +WORKDIR $USER_HOME + +# generate a script for Spack +RUN (echo "export SPACK_ROOT=$SPACK_ROOT" \ +&& echo ". $SPACK_ROOT/share/spack/setup-env.sh" \ +&& echo "") >> ~/.bashrc + +#------------------------------------------------------------------------------- +# Reset the entrypoint and CMD +#------------------------------------------------------------------------------- +ENTRYPOINT ["/bin/bash", "-l"] +CMD [] + +#----------------------------------------------------------------------- +# Build-time metadata as defined at http://label-schema.org +#----------------------------------------------------------------------- +ARG BUILD_DATE +ARG VCS_REF +ARG VCS_URL +LABEL org.label-schema.build-date=${BUILD_DATE} \ + org.label-schema.name="ANT-MOC CI image" \ + org.label-schema.description="Provides tools for ANT-MOC CI" \ + org.label-schema.vcs-ref=${VCS_REF} \ + org.label-schema.vcs-url=${VCS_URL} \ + org.label-schema.schema-version="0.1" diff --git a/Makefile b/Makefile index ee06dce..d08df95 100644 --- a/Makefile +++ b/Makefile @@ -33,13 +33,13 @@ GIT_COMMIT = $(strip $(shell git rev-parse --short HEAD)) # Targets to Build #=============================================================================== -.PHONY : docker_build docker_push output +.PHONY : build push output build_full push_full output_full default: build -build: docker_build output -release: docker_build docker_push output +release: build push output +release_full: build_full push_full output_full -docker_build: +build: # Build Docker image docker build \ --build-arg UBUNTU_CODE=$(UBUNTU_CODE) \ @@ -51,7 +51,7 @@ docker_build: --build-arg VCS_REF=$(GIT_COMMIT) \ -t $(DOCKER_IMAGE):$(DOCKER_TAG) . -docker_push: +push: # Tag image as latest docker tag $(DOCKER_IMAGE):$(DOCKER_TAG) $(DOCKER_IMAGE):latest @@ -61,3 +61,23 @@ docker_push: output: @echo Docker Image: $(DOCKER_IMAGE):$(DOCKER_TAG) + +build_full: + # Build Docker image with ROCm + docker build \ + --build-arg UBUNTU_CODE=$(UBUNTU_CODE) \ + --build-arg SPACK_VERSION=$(SPACK_VERSION) \ + --build-arg SPACK_IMAGE=$(SPACK_IMAGE) \ + --build-arg TARGET=$(TARGET) \ + --build-arg BUILD_DATE=$(BUILD_DATE) \ + --build-arg VCS_URL=$(VCS_URL) \ + --build-arg VCS_REF=$(GIT_COMMIT) \ + -f ./Dockerfile-full \ + -t $(DOCKER_IMAGE):$(DOCKER_TAG)-full . + +push_full: + # Push to DockerHub + docker push $(DOCKER_IMAGE):$(DOCKER_TAG)-full + +output_full: + @echo Docker Image: $(DOCKER_IMAGE):$(DOCKER_TAG)-full diff --git a/scripts/docker-slim.sh b/scripts/docker-slim.sh index d55f89f..0a69f70 100755 --- a/scripts/docker-slim.sh +++ b/scripts/docker-slim.sh @@ -14,5 +14,8 @@ slim build --target $IMAGE \ --include-path /opt/spack \ --include-path /opt/software \ --include-path /opt/repo \ + --exclude-pattern /usr/lib/x86_64-linux-gnu/dri \ + --preserve-path /tmp \ + --path-perms /usr/bin/sudo:4755 \ --entrypoint /bin/bash \ - --exec-file ./scripts/build-antmoc.sh + --exec-file ./scripts/test-antmoc.sh From 445daf4222e9cdcc32a298786817d3d99b381096 Mon Sep 17 00:00:00 2001 From: one Date: Tue, 9 Apr 2024 22:32:21 +0800 Subject: [PATCH 56/73] Try to fix the entrypoint for GitLab CI --- Dockerfile | 2 +- Dockerfile-full | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7b39da8..9dcab08 100644 --- a/Dockerfile +++ b/Dockerfile @@ -184,7 +184,7 @@ RUN (echo "export SPACK_ROOT=$SPACK_ROOT" \ #------------------------------------------------------------------------------- # Reset the entrypoint and CMD #------------------------------------------------------------------------------- -ENTRYPOINT ["/bin/bash", "-l"] +ENTRYPOINT ["/bin/bash", "-l", "-c"] CMD [] #----------------------------------------------------------------------- diff --git a/Dockerfile-full b/Dockerfile-full index a7ca151..b5bcca5 100644 --- a/Dockerfile-full +++ b/Dockerfile-full @@ -206,7 +206,7 @@ RUN (echo "export SPACK_ROOT=$SPACK_ROOT" \ #------------------------------------------------------------------------------- # Reset the entrypoint and CMD #------------------------------------------------------------------------------- -ENTRYPOINT ["/bin/bash", "-l"] +ENTRYPOINT ["/bin/bash", "-l", "-c"] CMD [] #----------------------------------------------------------------------- From be445f0e36ad3d7a119fc53a86690b017dedd128 Mon Sep 17 00:00:00 2001 From: one Date: Tue, 9 Apr 2024 23:48:06 +0800 Subject: [PATCH 57/73] Change entrypoint --- Dockerfile | 4 ++-- Dockerfile-full | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9dcab08..e618ffd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -184,8 +184,8 @@ RUN (echo "export SPACK_ROOT=$SPACK_ROOT" \ #------------------------------------------------------------------------------- # Reset the entrypoint and CMD #------------------------------------------------------------------------------- -ENTRYPOINT ["/bin/bash", "-l", "-c"] -CMD [] +ENTRYPOINT [] +CMD ["/bin/bash"] #----------------------------------------------------------------------- # Build-time metadata as defined at http://label-schema.org diff --git a/Dockerfile-full b/Dockerfile-full index b5bcca5..84b4d6b 100644 --- a/Dockerfile-full +++ b/Dockerfile-full @@ -206,8 +206,8 @@ RUN (echo "export SPACK_ROOT=$SPACK_ROOT" \ #------------------------------------------------------------------------------- # Reset the entrypoint and CMD #------------------------------------------------------------------------------- -ENTRYPOINT ["/bin/bash", "-l", "-c"] -CMD [] +ENTRYPOINT [] +CMD ["/bin/bash"] #----------------------------------------------------------------------- # Build-time metadata as defined at http://label-schema.org From a6031e12281adf17d63bb920b8961d3aaeac0c5c Mon Sep 17 00:00:00 2001 From: one Date: Wed, 10 Apr 2024 16:09:33 +0800 Subject: [PATCH 58/73] Use spack environments to build docker images --- .gitignore | 4 +- Dockerfile | 165 ++++++++++++----------------------------- scripts/docker-slim.sh | 1 - scripts/test-antmoc.sh | 23 +++--- spack.yaml | 32 ++++++++ 5 files changed, 95 insertions(+), 130 deletions(-) create mode 100644 spack.yaml diff --git a/.gitignore b/.gitignore index d312ee2..1bf1d62 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ __pycache__/ /mirror/ /ant-moc/ -/antmoc/ \ No newline at end of file +/antmoc/ +.spack-env/ +spack.lock \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e618ffd..788bde7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ ENV UBUNTU_CODE=${UBUNTU_CODE:-"jammy"} LABEL maintainer="An Wang " USER root -WORKDIR /tmp +WORKDIR /root #------------------------------------------------------------------------------- # Install system compilers and libraries @@ -17,7 +17,7 @@ RUN < $CONFIG_DIR/config.yaml -config: - install_tree: - root: $INSTALL_DIR - connect_timeout: 600 -EOF1 - -cat << EOF2 > $CONFIG_DIR/packages.yaml -packages: - all: - target: [$TARGET] -EOF2 - -spack mirror add --scope system local $MIRROR_DIR -spack repo add --scope system $REPO_DIR +mkdir -p ${CONFIG_DIR} +mkdir -p ${INSTALL_DIR} +mkdir -p ${ENV_DIR} +spack mirror add --scope system local ${MIRROR_DIR} +spack repo add --scope system ${REPO_DIR} EOF -#------------------------------------------------------------------------------- -# Find system compilers -#------------------------------------------------------------------------------- RUN < $CONFIG_DIR/compilers.yaml -compilers: -- compiler: - spec: clang@=CLANG_VERSION - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - flags: {} - operating_system: UBUNTU_VERSION - target: $TARGET - modules: [] - environment: {} - extra_rpaths: [] -EOF1 - -# substitute clang version and ubuntu version with the correct ones -sed -i -e "s/CLANG_VERSION/$(clang --version | grep -Po '(?<=version )[^ ]+')/g" $CONFIG_DIR/compilers.yaml -sed -i -e "s/UBUNTU_VERSION/$(spack debug report | grep -Po '(?<=linux-)[^-]+')/g" $CONFIG_DIR/compilers.yaml - -# find external packages +spack compiler find --scope system spack external find --scope system --not-buildable \ gcc \ autoconf \ @@ -123,69 +75,46 @@ spack external find --scope system --not-buildable \ python EOF - #------------------------------------------------------------------------------- # Install dependencies for antmoc #------------------------------------------------------------------------------- -# MPI specs -ARG MPICH_SPEC="mpich@=4.1.2~fortran" -ARG OPENMPI_SPEC="openmpi@=4.1.6" - # To avoid the default --reuse option of spack 0.21, -# add %clang and %gcc for every MPI spec. -RUN < /dev/null; then - useradd -m $USER_NAME -d $USER_HOME - echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers -fi -# copy spack configuration of the root to the new user's home -cp -r ~/.spack $USER_HOME/ -# change permissions (could be time-consuming) -chown -R $USER_NAME: $USER_HOME/.spack $CONFIG_DIR $REPO_DIR $MIRROR_DIR $INSTALL_DIR -EOF - -# transfer control to the default user -USER $USER_NAME -WORKDIR $USER_HOME - -# generate a script for Spack -RUN (echo "export SPACK_ROOT=$SPACK_ROOT" \ -&& echo ". $SPACK_ROOT/share/spack/setup-env.sh" \ -&& echo "") >> ~/.bashrc +# See https://github.com/open-mpi/ompi/pull/5598 +ENV OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 #------------------------------------------------------------------------------- -# Reset the entrypoint and CMD +# Reset the entrypoint or CMD +# In case it's broken, see /opt/spack/share/spack/docker/entrypoint.bash #------------------------------------------------------------------------------- -ENTRYPOINT [] -CMD ["/bin/bash"] +RUN < ~/setup-env.sh +chmod a+x ~/setup-env.sh +EOF + +# This is what GitLab CI needs +ENTRYPOINT ["/bin/bash", "/usr/local/bin/spack-env"] +CMD ["interactive-shell"] #----------------------------------------------------------------------- # Build-time metadata as defined at http://label-schema.org @@ -194,8 +123,8 @@ ARG BUILD_DATE ARG VCS_REF ARG VCS_URL LABEL org.label-schema.build-date=${BUILD_DATE} \ - org.label-schema.name="ANT-MOC CI image" \ - org.label-schema.description="Provides tools for ANT-MOC CI" \ - org.label-schema.vcs-ref=${VCS_REF} \ - org.label-schema.vcs-url=${VCS_URL} \ - org.label-schema.schema-version="0.1" + org.label-schema.name="ANT-MOC CI image" \ + org.label-schema.description="Provides tools for ANT-MOC CI" \ + org.label-schema.vcs-ref=${VCS_REF} \ + org.label-schema.vcs-url=${VCS_URL} \ + org.label-schema.schema-version="0.1" diff --git a/scripts/docker-slim.sh b/scripts/docker-slim.sh index 0a69f70..eb174c5 100755 --- a/scripts/docker-slim.sh +++ b/scripts/docker-slim.sh @@ -17,5 +17,4 @@ slim build --target $IMAGE \ --exclude-pattern /usr/lib/x86_64-linux-gnu/dri \ --preserve-path /tmp \ --path-perms /usr/bin/sudo:4755 \ - --entrypoint /bin/bash \ --exec-file ./scripts/test-antmoc.sh diff --git a/scripts/test-antmoc.sh b/scripts/test-antmoc.sh index 4dcaade..0442f84 100755 --- a/scripts/test-antmoc.sh +++ b/scripts/test-antmoc.sh @@ -1,8 +1,13 @@ -#!/usr/bin/env bash # For slim toolkit 1.40.11 -sudo -Hi -u hpcer bash -i << EOF +sudo -Hi bash -i << EOF set -e whoami +env + +# Setup environment +. ~/setup-env.sh +spack debug report +spack find -v antmoc WORKDIR=/tmp/ant-moc [ -d \$WORKDIR ] && rm -rf \$WORKDIR @@ -13,11 +18,6 @@ mkdir \$WORKDIR cp -r /opt/mnt/ant-moc/. \$WORKDIR/ cd \$WORKDIR -# Setup environment -source ~/.bashrc -spack debug report -spack find -v antmoc - # Compilers in tuple (C compiler, C++ compiler, spack spec) declare -A COMPILERS=( \ ["gcc"]="gcc g++ %gcc" \ @@ -31,10 +31,13 @@ declare -A MPIS=( \ ["openmpi"]="+mpi^openmpi") # Test cases +# declare -a TESTS=( \ +# "gcc serial run" "gcc mpich run" "gcc openmpi run" \ +# "clang serial run" "clang mpich run" \ +# "hipcc serial build" "hipcc mpich build" "hipcc openmpi build" ) declare -a TESTS=( \ "gcc serial run" "gcc mpich run" "gcc openmpi run" \ - "clang serial run" "clang mpich run" \ - "hipcc serial build" "hipcc mpich build" "hipcc openmpi build" ) + "clang serial run" "clang mpich run" ) # Run tests for s in "\${TESTS[@]}"; do @@ -108,4 +111,4 @@ done cd rm -rf \$WORKDIR -EOF +EOF \ No newline at end of file diff --git a/spack.yaml b/spack.yaml new file mode 100644 index 0000000..404d272 --- /dev/null +++ b/spack.yaml @@ -0,0 +1,32 @@ +# This is a Spack Environment file. +# +# It describes a set of packages to be installed, along with +# configuration settings. +spack: + # add package specs to the `specs` list + specs: + - cmake%gcc + - lcov@=1.16%gcc + - antmoc%clang~mpi + - antmoc%clang+mpi ^mpich@=4.1.2%clang~fortran + - antmoc%gcc~mpi + - antmoc%gcc+mpi ^mpich@=4.1.2%gcc~fortran + - antmoc%gcc+mpi ^openmpi@=4.1.6%gcc + view: false + concretizer: + unify: when_possible + + config: + install_tree: /opt/software + connect_timeout: 600 + + packages: + all: + target: [x86_64] + + container: + format: docker + images: + os: "ubuntu:22.04" + spack: 0.21.2 + strip: true From 6de31d6f684aeac659732ca300f0eb47b8900077 Mon Sep 17 00:00:00 2001 From: one Date: Wed, 10 Apr 2024 17:58:29 +0800 Subject: [PATCH 59/73] Generate tests with a python script --- scripts/docker-slim.sh | 1 + scripts/generate-test.py | 85 +++++++++++++++++++++++++++++++++++++ scripts/test-antmoc.sh | 91 ++++------------------------------------ 3 files changed, 95 insertions(+), 82 deletions(-) create mode 100755 scripts/generate-test.py diff --git a/scripts/docker-slim.sh b/scripts/docker-slim.sh index eb174c5..66a471c 100755 --- a/scripts/docker-slim.sh +++ b/scripts/docker-slim.sh @@ -5,6 +5,7 @@ IMAGE=${1:-"antmoc/antmoc-ci:0.1.16-alpha"} # /opt is the largest directory slim build --target $IMAGE \ --mount $(pwd)/ant-moc:/opt/mnt/ant-moc \ + --mount $(pwd)/scripts:/opt/mnt/scripts \ --http-probe=false \ --show-clogs \ --include-path /root \ diff --git a/scripts/generate-test.py b/scripts/generate-test.py new file mode 100755 index 0000000..cf69a41 --- /dev/null +++ b/scripts/generate-test.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# ------------------------------------------------------------------------------- +# Preparing +# +# This script depends on Spack to validate specs. Jobs will be generated only +# for valid specs. +# ------------------------------------------------------------------------------- +import sys, getopt + +# parse command line arguments +opts, args = getopt.getopt(sys.argv[1:], "h", ["job="]) + +for name, value in opts: + if name == "--job": + job_spec = value + +# ------------------------------------------------------------------------------- +# Helpers +# ------------------------------------------------------------------------------- +# tuples for (CXX compiler, compiler specs) +cc_specs = { + "gcc": ("g++", "%gcc"), + "clang": ("clang++", "%clang"), + "hipcc": ("hipcc", "%gcc"), +} + +# mpi specs +mpi_specs = {"serial": "~mpi", "mpich": "+mpi^mpich", "openmpi": "+mpi^openmpi"} + + +def on_off(bool_value): + return "ON" if bool_value else "OFF" + + +class Job: + def __init__(self, compiler, mpi, stage): + self.cc = compiler + self.cxx = cc_specs[compiler][0] + self.cc_spec = cc_specs[compiler][1] + self.mpi = mpi + self.mpi_spec = mpi_specs[mpi] + self.stage = stage + + self.use_mpi = self.mpi != "serial" + self.use_hip = self.cc == "hipcc" + self.random_test = self.mpi == "serial" # if no mpi + self.use_spec = f"antmoc {self.cc_spec} {self.mpi_spec}" + + def __str__(self): + return f"({self.cc}, {self.mpi}, <={self.stage})" + + def __repr__(self): + return self.__str__() + + +# ------------------------------------------------------------------------------- +# Generating +# ------------------------------------------------------------------------------- +job_spec = job_spec.split() +job = Job(job_spec[0], job_spec[1], job_spec[2]) + +print( + f"""set -e +spack unload --all +spack load cmake%gcc {job.use_spec} +spack find --loaded + +rm -rf build/ &> /dev/null + +cmake -S . -B build \\ + -DCMAKE_C_COMPILER={job.cc} \\ + -DCMAKE_CXX_COMPILER={job.cxx} \\ + -DCMAKE_BUILD_TYPE=Release \\ + -DBUILD_SHARED_LIBS:BOOL=ON \\ + -DENABLE_ALL_WARNINGS=OFF \\ + -DENABLE_TESTS:BOOL=ON \\ + -DENABLE_MPI:BOOL={job.use_mpi} \\ + -DENABLE_HIP:BOOL={job.use_hip} + +cmake --build build -j$(nproc) +""" +) + +if job.stage in ["run", "install"]: + print(f"ctest --test-dir build --output-on-failure {'--schedule-random' if job.random_test else ''}") \ No newline at end of file diff --git a/scripts/test-antmoc.sh b/scripts/test-antmoc.sh index 0442f84..29af996 100755 --- a/scripts/test-antmoc.sh +++ b/scripts/test-antmoc.sh @@ -10,25 +10,16 @@ spack debug report spack find -v antmoc WORKDIR=/tmp/ant-moc -[ -d \$WORKDIR ] && rm -rf \$WORKDIR -mkdir \$WORKDIR +[ -d \$WORKDIR ] && rm -rf \$WORKDIR && mkdir \$WORKDIR # Always mount ANT-MOC to this directory in containers # -v ./ant-moc:/opt/mnt/ant-moc cp -r /opt/mnt/ant-moc/. \$WORKDIR/ cd \$WORKDIR -# Compilers in tuple (C compiler, C++ compiler, spack spec) -declare -A COMPILERS=( \ - ["gcc"]="gcc g++ %gcc" \ - ["clang"]="clang clang++ %clang" \ - ["hipcc"]="hipcc hipcc %gcc") - -# MPI specs -declare -A MPIS=( \ - ["serial"]="~mpi" \ - ["mpich"]="+mpi^mpich" \ - ["openmpi"]="+mpi^openmpi") +# Always mount scripts to this directory +# -v ./scripts:/opt/mnt/scripts +GENERATOR=/opt/mnt/scripts/generate-test.py # Test cases # declare -a TESTS=( \ @@ -40,75 +31,11 @@ declare -a TESTS=( \ "clang serial run" "clang mpich run" ) # Run tests -for s in "\${TESTS[@]}"; do - cd \$WORKDIR - - declare -a test=(\${s[@]}) - declare -a cc=(\${COMPILERS[\${test[0]}]}) - mpi=\${test[1]} - stage=\${test[2]} - - echo "\${cc[@]} \$mpi \$stage" - - C_COMPILER="\${cc[0]}" - CXX_COMPILER="\${cc[1]}" - USE_SPECS="antmoc \${cc[2]} \${MPIS[\$mpi]}" - BUILD_TYPE=Release - BUILD_SHARED_LIBS=ON - ENABLE_ALL_WARNINGS=OFF - - # Enable tests - if [ \$stage == "run" ]; then - ENABLE_TESTS=ON - else - ENABLE_TESTS=OFF - fi - - # Enable MPI or not - if [ \$mpi == "serial" ]; then - ENABLE_MPI=OFF - CTEST_RANDOM=ON - else - ENABLE_MPI=ON - CTEST_RANDOM=OFF - fi - - # Enable HIP or not - if [ \${cc[0]} == "hipcc" ]; then - ENABLE_HIP=ON - else - ENABLE_HIP=OFF - fi - - # Load dependencies - spack unload --all - spack load cmake%gcc \$USE_SPECS - spack find --loaded - - rm -rf build/ &> /dev/null - - # Build ANT-MOC - cmake -S . -B build \ - -DCMAKE_C_COMPILER=\$C_COMPILER \ - -DCMAKE_CXX_COMPILER=\$CXX_COMPILER \ - -DCMAKE_BUILD_TYPE=\$BUILD_TYPE \ - -DBUILD_SHARED_LIBS:BOOL=\$BUILD_SHARED_LIBS \ - -DENABLE_ALL_WARNINGS=\$ENABLE_ALL_WARNINGS \ - -DENABLE_TESTS:BOOL=\$ENABLE_TESTS \ - -DENABLE_MPI:BOOL=\$ENABLE_MPI \ - -DENABLE_HIP:BOOL=\$ENABLE_HIP - - echo -e "Building ANT-MOC..." - cmake --build build -j\$(nproc) - - if [ \$stage == "run" ]; then - ARGS="--output-on-failure" - if [ "\$CTEST_RANDOM" == "ON" ]; then ARGS="\$ARGS --schedule-random"; fi - cd build/ - ctest \$ARGS - fi +for jobspec in "\${TESTS[@]}"; do + bash << EOF1 + \$(python3 \${GENERATOR} --job "\${jobspec}") +EOF1 done -cd -rm -rf \$WORKDIR +cd && rm -rf \$WORKDIR EOF \ No newline at end of file From bc6d3ec714df4b4a5b122dfb05773f59b6a884be Mon Sep 17 00:00:00 2001 From: one Date: Wed, 10 Apr 2024 17:58:47 +0800 Subject: [PATCH 60/73] Add a set-up script to the image --- Dockerfile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 788bde7..a73b6e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -98,17 +98,20 @@ RUN find -L "${INSTALL_DIR}" -type f -exec readlink -f '{}' \; | \ awk -F: '{print $1}' | \ xargs strip -s -# See https://github.com/open-mpi/ompi/pull/5598 -ENV OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 - #------------------------------------------------------------------------------- # Reset the entrypoint or CMD # In case it's broken, see /opt/spack/share/spack/docker/entrypoint.bash #------------------------------------------------------------------------------- RUN < ~/setup-env.sh +cat << EOF1 > ~/setup-env.sh +# Spack +. ${SPACK_ROOT}/share/spack/setup-env.sh +# Spack environment +$(spack env activate --sh -d ${ENV_DIR}) +# See https://github.com/open-mpi/ompi/pull/5598 +export OMPI_ALLOW_RUN_AS_ROOT=1 +export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 +EOF1 chmod a+x ~/setup-env.sh EOF From 8e6ab84a7f932ac9c1ab7b45e167e066d2318df0 Mon Sep 17 00:00:00 2001 From: one Date: Wed, 10 Apr 2024 18:59:22 +0800 Subject: [PATCH 61/73] Add tag to slimmed image --- scripts/docker-slim.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/docker-slim.sh b/scripts/docker-slim.sh index 66a471c..181ed24 100755 --- a/scripts/docker-slim.sh +++ b/scripts/docker-slim.sh @@ -1,8 +1,9 @@ # See https://github.com/slimtoolkit/slim IMAGE=${1:-"antmoc/antmoc-ci:0.1.16-alpha"} +NEWTAG=${IMAGE%-*} -# /opt is the largest directory +/opt is the largest directory slim build --target $IMAGE \ --mount $(pwd)/ant-moc:/opt/mnt/ant-moc \ --mount $(pwd)/scripts:/opt/mnt/scripts \ @@ -18,4 +19,5 @@ slim build --target $IMAGE \ --exclude-pattern /usr/lib/x86_64-linux-gnu/dri \ --preserve-path /tmp \ --path-perms /usr/bin/sudo:4755 \ + --tag $NEWTAG \ --exec-file ./scripts/test-antmoc.sh From a1da9d26d7b7eff096d692b6268cae933773ea91 Mon Sep 17 00:00:00 2001 From: one Date: Wed, 10 Apr 2024 19:04:35 +0800 Subject: [PATCH 62/73] Update docker-slim.sh --- scripts/docker-slim.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/docker-slim.sh b/scripts/docker-slim.sh index 181ed24..a1b03c8 100755 --- a/scripts/docker-slim.sh +++ b/scripts/docker-slim.sh @@ -4,7 +4,7 @@ IMAGE=${1:-"antmoc/antmoc-ci:0.1.16-alpha"} NEWTAG=${IMAGE%-*} /opt is the largest directory -slim build --target $IMAGE \ +slim build \ --mount $(pwd)/ant-moc:/opt/mnt/ant-moc \ --mount $(pwd)/scripts:/opt/mnt/scripts \ --http-probe=false \ @@ -19,5 +19,6 @@ slim build --target $IMAGE \ --exclude-pattern /usr/lib/x86_64-linux-gnu/dri \ --preserve-path /tmp \ --path-perms /usr/bin/sudo:4755 \ + --target $IMAGE \ --tag $NEWTAG \ --exec-file ./scripts/test-antmoc.sh From d5e79e1fa53d21b368be6d4dc6a1bddf1b59c42f Mon Sep 17 00:00:00 2001 From: one Date: Wed, 10 Apr 2024 19:46:11 +0800 Subject: [PATCH 63/73] Generate installation tests --- scripts/generate-test.py | 7 ++++++- scripts/test-antmoc.sh | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/generate-test.py b/scripts/generate-test.py index cf69a41..a4fa4fd 100755 --- a/scripts/generate-test.py +++ b/scripts/generate-test.py @@ -82,4 +82,9 @@ def __repr__(self): ) if job.stage in ["run", "install"]: - print(f"ctest --test-dir build --output-on-failure {'--schedule-random' if job.random_test else ''}") \ No newline at end of file + print( + f"ctest --test-dir build --output-on-failure {'--schedule-random' if job.random_test else ''}" + ) + +if job.stage in ["install"]: + print(f"cmake --install build && ldd antmoc") diff --git a/scripts/test-antmoc.sh b/scripts/test-antmoc.sh index 29af996..0e7b915 100755 --- a/scripts/test-antmoc.sh +++ b/scripts/test-antmoc.sh @@ -27,8 +27,8 @@ GENERATOR=/opt/mnt/scripts/generate-test.py # "clang serial run" "clang mpich run" \ # "hipcc serial build" "hipcc mpich build" "hipcc openmpi build" ) declare -a TESTS=( \ - "gcc serial run" "gcc mpich run" "gcc openmpi run" \ - "clang serial run" "clang mpich run" ) + "gcc serial install" "gcc mpich install" "gcc openmpi install" \ + "clang serial install" "clang mpich install" ) # Run tests for jobspec in "\${TESTS[@]}"; do From 962a9cb52b0d3cd30bb52252759b38d65b44eba5 Mon Sep 17 00:00:00 2001 From: one Date: Thu, 11 Apr 2024 12:43:53 +0800 Subject: [PATCH 64/73] Update spack package, use cmake preset to test antmoc --- repo/packages/antmoc/package.py | 11 +++++--- scripts/docker-slim.sh | 2 +- scripts/generate-test.py | 48 +++++++++++---------------------- 3 files changed, 24 insertions(+), 37 deletions(-) diff --git a/repo/packages/antmoc/package.py b/repo/packages/antmoc/package.py index a09bfc5..070af71 100644 --- a/repo/packages/antmoc/package.py +++ b/repo/packages/antmoc/package.py @@ -10,16 +10,19 @@ class Antmoc(BundlePackage): license("MIT") + version('develop') version('0.1.15') + version('0.1.14') variant('mpi', default=False, description='Enable MPI support') depends_on('cmake@3.16:', type='build') - depends_on('mpi@=3.0', when='+mpi', type=('build', 'link', 'run')) + depends_on('mpi@3', when='+mpi', type=('build', 'link', 'run')) depends_on('cxxopts@3') - depends_on('fmt@8:10 +shared') + depends_on('fmt@6 +shared', when='@:0.1.15') + depends_on('fmt@8:10 +shared', when='@0.1.16:') depends_on('tinyxml2@7:10 +shared') depends_on('toml11@3.6:3.7') - depends_on('hdf5@1.12:1.14 ~mpi+shared', when='~mpi') - depends_on('hdf5@1.12:1.14 +mpi+shared', when='+mpi') + depends_on('hdf5@1.10:1.14 ~mpi+shared', when='~mpi') + depends_on('hdf5@1.10:1.14 +mpi+shared', when='+mpi') depends_on('googletest@1.10.0: +gmock+pthreads+shared') diff --git a/scripts/docker-slim.sh b/scripts/docker-slim.sh index a1b03c8..9252136 100755 --- a/scripts/docker-slim.sh +++ b/scripts/docker-slim.sh @@ -1,7 +1,7 @@ # See https://github.com/slimtoolkit/slim IMAGE=${1:-"antmoc/antmoc-ci:0.1.16-alpha"} -NEWTAG=${IMAGE%-*} +NEWTAG=${IMAGE%-*}-slim /opt is the largest directory slim build \ diff --git a/scripts/generate-test.py b/scripts/generate-test.py index a4fa4fd..d8a2478 100755 --- a/scripts/generate-test.py +++ b/scripts/generate-test.py @@ -17,34 +17,28 @@ # ------------------------------------------------------------------------------- # Helpers # ------------------------------------------------------------------------------- -# tuples for (CXX compiler, compiler specs) +# compiler specs cc_specs = { - "gcc": ("g++", "%gcc"), - "clang": ("clang++", "%clang"), - "hipcc": ("hipcc", "%gcc"), + "gcc": "%gcc", + "clang": "%clang", + "hipcc": "%gcc", } # mpi specs mpi_specs = {"serial": "~mpi", "mpich": "+mpi^mpich", "openmpi": "+mpi^openmpi"} -def on_off(bool_value): - return "ON" if bool_value else "OFF" - - class Job: - def __init__(self, compiler, mpi, stage): - self.cc = compiler - self.cxx = cc_specs[compiler][0] - self.cc_spec = cc_specs[compiler][1] + def __init__(self, cc, mpi, stage): + self.cc = cc self.mpi = mpi - self.mpi_spec = mpi_specs[mpi] self.stage = stage - self.use_mpi = self.mpi != "serial" - self.use_hip = self.cc == "hipcc" - self.random_test = self.mpi == "serial" # if no mpi - self.use_spec = f"antmoc {self.cc_spec} {self.mpi_spec}" + # preset name: compiler?-mpi?-debug + self.preset = f"{cc}-{'serial' if mpi=='serial' else 'mpi'}-debug" + self.build_dir = f"build/{self.preset}" + self.use_spec = f"antmoc {cc_specs[cc]} {mpi_specs[mpi]}" + self.random_test = "ON" if self.mpi == "serial" else "OFF" # no mpi def __str__(self): return f"({self.cc}, {self.mpi}, <={self.stage})" @@ -65,26 +59,16 @@ def __repr__(self): spack load cmake%gcc {job.use_spec} spack find --loaded -rm -rf build/ &> /dev/null - -cmake -S . -B build \\ - -DCMAKE_C_COMPILER={job.cc} \\ - -DCMAKE_CXX_COMPILER={job.cxx} \\ - -DCMAKE_BUILD_TYPE=Release \\ - -DBUILD_SHARED_LIBS:BOOL=ON \\ - -DENABLE_ALL_WARNINGS=OFF \\ - -DENABLE_TESTS:BOOL=ON \\ - -DENABLE_MPI:BOOL={job.use_mpi} \\ - -DENABLE_HIP:BOOL={job.use_hip} - -cmake --build build -j$(nproc) +rm -rf {job.build_dir} &> /dev/null +cmake --preset {job.preset} +cmake --build {job.build_dir} -j$(nproc) """ ) if job.stage in ["run", "install"]: print( - f"ctest --test-dir build --output-on-failure {'--schedule-random' if job.random_test else ''}" + f"ctest --test-dir {job.build_dir} --output-on-failure {'--schedule-random' if job.random_test else ''}" ) if job.stage in ["install"]: - print(f"cmake --install build && ldd antmoc") + print(f"cmake --install {job.build_dir} && ldd antmoc") From 7d646970acd2be29485facac3165289551f66ab8 Mon Sep 17 00:00:00 2001 From: one Date: Thu, 11 Apr 2024 18:49:55 +0800 Subject: [PATCH 65/73] Add valgrind --- repo/packages/antmoc/package.py | 2 +- scripts/generate-test.py | 2 +- spack.yaml | 11 ++++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/repo/packages/antmoc/package.py b/repo/packages/antmoc/package.py index 070af71..832cd70 100644 --- a/repo/packages/antmoc/package.py +++ b/repo/packages/antmoc/package.py @@ -10,7 +10,7 @@ class Antmoc(BundlePackage): license("MIT") - version('develop') + version('develop', preferred=True) version('0.1.15') version('0.1.14') diff --git a/scripts/generate-test.py b/scripts/generate-test.py index d8a2478..bf04c66 100755 --- a/scripts/generate-test.py +++ b/scripts/generate-test.py @@ -71,4 +71,4 @@ def __repr__(self): ) if job.stage in ["install"]: - print(f"cmake --install {job.build_dir} && ldd antmoc") + print(f"cmake --install {job.build_dir} && ldd $(which antmoc)") diff --git a/spack.yaml b/spack.yaml index 404d272..d45f48c 100644 --- a/spack.yaml +++ b/spack.yaml @@ -7,11 +7,12 @@ spack: specs: - cmake%gcc - lcov@=1.16%gcc - - antmoc%clang~mpi - - antmoc%clang+mpi ^mpich@=4.1.2%clang~fortran - - antmoc%gcc~mpi - - antmoc%gcc+mpi ^mpich@=4.1.2%gcc~fortran - - antmoc%gcc+mpi ^openmpi@=4.1.6%gcc + - valgrind@3.20%gcc + - antmoc@develop%clang~mpi + - antmoc@develop%clang+mpi ^mpich@=4.1.2%clang~fortran + - antmoc@develop%gcc~mpi + - antmoc@develop%gcc+mpi ^mpich@=4.1.2%gcc~fortran + - antmoc@develop%gcc+mpi ^openmpi@=4.1.6%gcc view: false concretizer: unify: when_possible From e5cb3aeb3878cc56a2972e9d71ceb73be6929a7f Mon Sep 17 00:00:00 2001 From: one Date: Thu, 11 Apr 2024 23:08:25 +0800 Subject: [PATCH 66/73] Add gcovr, fix valgrind version --- Dockerfile | 1 + spack.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a73b6e9..36898ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,7 @@ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ libomp-15-dev \ sudo \ cmake \ + gcovr \ openssh-server apt-get clean rm -rf /var/lib/apt/lists/* diff --git a/spack.yaml b/spack.yaml index d45f48c..96ff0f2 100644 --- a/spack.yaml +++ b/spack.yaml @@ -7,7 +7,7 @@ spack: specs: - cmake%gcc - lcov@=1.16%gcc - - valgrind@3.20%gcc + - valgrind@3.20%gcc ^mpich@=4.1.2%gcc~fortran - antmoc@develop%clang~mpi - antmoc@develop%clang+mpi ^mpich@=4.1.2%clang~fortran - antmoc@develop%gcc~mpi From e0acd5724c289d93798b472b2a1d03b5e0cb3e67 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 12 Apr 2024 11:50:54 +0800 Subject: [PATCH 67/73] Add libc6-dbg for valgrind --- Dockerfile | 1 + scripts/docker-slim.sh | 2 +- scripts/test-antmoc.sh | 3 +-- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 36898ca..ee88adb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ llvm-15 \ clang-15 \ libomp-15-dev \ + libc6-dbg \ sudo \ cmake \ gcovr \ diff --git a/scripts/docker-slim.sh b/scripts/docker-slim.sh index 9252136..42488e0 100755 --- a/scripts/docker-slim.sh +++ b/scripts/docker-slim.sh @@ -17,7 +17,7 @@ slim build \ --include-path /opt/software \ --include-path /opt/repo \ --exclude-pattern /usr/lib/x86_64-linux-gnu/dri \ - --preserve-path /tmp \ + --preserve-path /root \ --path-perms /usr/bin/sudo:4755 \ --target $IMAGE \ --tag $NEWTAG \ diff --git a/scripts/test-antmoc.sh b/scripts/test-antmoc.sh index 0e7b915..1eccb24 100755 --- a/scripts/test-antmoc.sh +++ b/scripts/test-antmoc.sh @@ -9,7 +9,7 @@ env spack debug report spack find -v antmoc -WORKDIR=/tmp/ant-moc +WORKDIR=/root/ant-moc [ -d \$WORKDIR ] && rm -rf \$WORKDIR && mkdir \$WORKDIR # Always mount ANT-MOC to this directory in containers @@ -37,5 +37,4 @@ for jobspec in "\${TESTS[@]}"; do EOF1 done -cd && rm -rf \$WORKDIR EOF \ No newline at end of file From 8c8780acb88d101534ce4ab0a3156b30442c6f39 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 12 Apr 2024 13:57:13 +0800 Subject: [PATCH 68/73] Clean up test history --- scripts/test-antmoc.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/test-antmoc.sh b/scripts/test-antmoc.sh index 1eccb24..a22d54c 100755 --- a/scripts/test-antmoc.sh +++ b/scripts/test-antmoc.sh @@ -37,4 +37,6 @@ for jobspec in "\${TESTS[@]}"; do EOF1 done +cd && rm -rf \$WORKDIR +history -c EOF \ No newline at end of file From f6c14fbc1d65b7c319a4ac95d610a113b17563a9 Mon Sep 17 00:00:00 2001 From: one Date: Fri, 12 Apr 2024 20:01:50 +0800 Subject: [PATCH 69/73] Update package --- repo/packages/antmoc/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo/packages/antmoc/package.py b/repo/packages/antmoc/package.py index 832cd70..804f275 100644 --- a/repo/packages/antmoc/package.py +++ b/repo/packages/antmoc/package.py @@ -19,7 +19,7 @@ class Antmoc(BundlePackage): depends_on('cmake@3.16:', type='build') depends_on('mpi@3', when='+mpi', type=('build', 'link', 'run')) depends_on('cxxopts@3') - depends_on('fmt@6 +shared', when='@:0.1.15') + depends_on('fmt@6:8 +shared', when='@:0.1.15') depends_on('fmt@8:10 +shared', when='@0.1.16:') depends_on('tinyxml2@7:10 +shared') depends_on('toml11@3.6:3.7') From 73bf1b117c7f978b2f7681b1e563360eb260e85f Mon Sep 17 00:00:00 2001 From: one Date: Fri, 12 Apr 2024 23:30:58 +0800 Subject: [PATCH 70/73] Add OCI labels --- Dockerfile | 20 ++++++++-------- Dockerfile-full | 20 ++++++++-------- Makefile | 16 ++++++------- README.md | 62 +++++-------------------------------------------- 4 files changed, 34 insertions(+), 84 deletions(-) diff --git a/Dockerfile b/Dockerfile index ee88adb..8a98a9c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -122,14 +122,14 @@ ENTRYPOINT ["/bin/bash", "/usr/local/bin/spack-env"] CMD ["interactive-shell"] #----------------------------------------------------------------------- -# Build-time metadata as defined at http://label-schema.org +# OCI annotations #----------------------------------------------------------------------- -ARG BUILD_DATE -ARG VCS_REF -ARG VCS_URL -LABEL org.label-schema.build-date=${BUILD_DATE} \ - org.label-schema.name="ANT-MOC CI image" \ - org.label-schema.description="Provides tools for ANT-MOC CI" \ - org.label-schema.vcs-ref=${VCS_REF} \ - org.label-schema.vcs-url=${VCS_URL} \ - org.label-schema.schema-version="0.1" +ARG OCI_CREATED +ARG OCI_REVISION +ARG OCI_SOURCE +LABEL org.opencontainers.image.created=${OCI_CREATED} \ + org.opencontainers.image.source=${OCI_SOURCE} \ + org.opencontainers.image.revision=${OCI_REVISION} \ + org.opencontainers.image.title="ANT-MOC CI image" \ + org.opencontainers.image.description="Tools for ANT-MOC CI pipelines" \ + org.opencontainers.image.url="https://hub.docker.com/r/antmoc/antmoc-ci" diff --git a/Dockerfile-full b/Dockerfile-full index 84b4d6b..e58a404 100644 --- a/Dockerfile-full +++ b/Dockerfile-full @@ -210,14 +210,14 @@ ENTRYPOINT [] CMD ["/bin/bash"] #----------------------------------------------------------------------- -# Build-time metadata as defined at http://label-schema.org +# OCI annotations #----------------------------------------------------------------------- -ARG BUILD_DATE -ARG VCS_REF -ARG VCS_URL -LABEL org.label-schema.build-date=${BUILD_DATE} \ - org.label-schema.name="ANT-MOC CI image" \ - org.label-schema.description="Provides tools for ANT-MOC CI" \ - org.label-schema.vcs-ref=${VCS_REF} \ - org.label-schema.vcs-url=${VCS_URL} \ - org.label-schema.schema-version="0.1" + ARG OCI_CREATED + ARG OCI_REVISION + ARG OCI_SOURCE + LABEL org.opencontainers.image.created=${OCI_CREATED} \ + org.opencontainers.image.source=${OCI_SOURCE} \ + org.opencontainers.image.revision=${OCI_REVISION} \ + org.opencontainers.image.title="ANT-MOC CI image" \ + org.opencontainers.image.description="Tools for ANT-MOC CI pipelines" \ + org.opencontainers.image.url="https://hub.docker.com/r/antmoc/antmoc-ci" diff --git a/Makefile b/Makefile index d08df95..245b5c6 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,8 @@ USER_NAME ?= hpcer # Variables and objects #=============================================================================== -BUILD_DATE=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") -VCS_URL=$(shell git config --get remote.origin.url) +OCI_CREATED=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") +OCI_SOURCE=$(shell git config --get remote.origin.url) # Get the latest commit GIT_COMMIT = $(strip $(shell git rev-parse --short HEAD)) @@ -46,9 +46,9 @@ build: --build-arg SPACK_VERSION=$(SPACK_VERSION) \ --build-arg SPACK_IMAGE=$(SPACK_IMAGE) \ --build-arg TARGET=$(TARGET) \ - --build-arg BUILD_DATE=$(BUILD_DATE) \ - --build-arg VCS_URL=$(VCS_URL) \ - --build-arg VCS_REF=$(GIT_COMMIT) \ + --build-arg OCI_CREATED=$(OCI_CREATED) \ + --build-arg OCI_SOURCE=$(OCI_SOURCE) \ + --build-arg OCI_REVISION=$(GIT_COMMIT) \ -t $(DOCKER_IMAGE):$(DOCKER_TAG) . push: @@ -69,9 +69,9 @@ build_full: --build-arg SPACK_VERSION=$(SPACK_VERSION) \ --build-arg SPACK_IMAGE=$(SPACK_IMAGE) \ --build-arg TARGET=$(TARGET) \ - --build-arg BUILD_DATE=$(BUILD_DATE) \ - --build-arg VCS_URL=$(VCS_URL) \ - --build-arg VCS_REF=$(GIT_COMMIT) \ + --build-arg OCI_CREATED=$(OCI_CREATED) \ + --build-arg OCI_SOURCE=$(OCI_SOURCE) \ + --build-arg OCI_REVISION=$(GIT_COMMIT) \ -f ./Dockerfile-full \ -t $(DOCKER_IMAGE):$(DOCKER_TAG)-full . diff --git a/README.md b/README.md index f71ab02..d2c7ee5 100644 --- a/README.md +++ b/README.md @@ -1,64 +1,14 @@ ANT-MOC CI ========== -[![Commit](https://images.microbadger.com/badges/commit/antmoc/antmoc-ci.svg)](https://github.com/antmoc-bot/antmoc-ci) -[![Version](https://images.microbadger.com/badges/version/antmoc/antmoc-ci.svg)](https://hub.docker.com/repository/docker/antmoc/antmoc-ci) -[![Docker Pulls](https://img.shields.io/docker/pulls/antmoc/antmoc-ci?color=informational)](https://hub.docker.com/repository/docker/antmoc/antmoc-ci) -[![Layers](https://images.microbadger.com/badges/image/antmoc/antmoc-ci.svg)](https://microbadger.com/images/antmoc/antmoc-ci) -[![Automated Build](https://img.shields.io/docker/automated/antmoc/antmoc-ci)](https://hub.docker.com/repository/docker/antmoc/antmoc-ci) +[![Docker Image Version](https://img.shields.io/docker/v/antmoc/antmoc-ci)](https://hub.docker.com/r/antmoc/antmoc-ci/tags) +![Docker Image Size](https://img.shields.io/docker/image-size/antmoc/antmoc-ci) +![Docker Pulls](https://img.shields.io/docker/pulls/antmoc/antmoc-ci?color=informational) +![Automated Build](https://img.shields.io/docker/automated/antmoc/antmoc-ci) -This project provides official ANT-MOC CI images starting from v0.1.15. +Official ANT-MOC CI images starting from 0.1.15. -## Supported tags - -- `latest` -- `0.1.15` - -> Legacy images (before v0.1.15) could be found at +> Legacy images (before 0.1.15) could be found at > > - DockerHub: [leavesask/antmoc-ci](https://hub.docker.com/r/leavesask/antmoc-ci) > - GitHub: [docker-antmoc-ci](https://github.com/alephpiece/docker-antmoc-ci) - -## How to use - -1. [Install docker engine](https://docs.docker.com/install/) - -2. Pull the image - ```bash - docker pull antmoc/antmoc-ci: - ``` - -3. Run the image interactively - ```bash - docker run -it --rm antmoc/antmoc-ci: - ``` - -## How to build - -The base image is [Spack](https://hub.docker.com/r/spack). - -### make - -It is highly recommended that you build the image with `make`. - -```bash -# Build an image for code coverage -make - -# Build and publish the image -make release -``` - -### docker build - -As an alternative, you can build the image with `docker build` command. - -```bash -docker build \ - --build-arg SPACK_IMAGE="spack/ubuntu-bionic" \ - --build-arg SPACK_VERSION="latest" \ - --build-arg BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \ - --build-arg VCS_REF=$(git rev-parse --short HEAD) \ - -t antmoc/antmoc-ci:latest . -``` - From 08952d30b773b576564aae52b4aef74664d7e0fc Mon Sep 17 00:00:00 2001 From: one Date: Sat, 13 Apr 2024 00:05:45 +0800 Subject: [PATCH 71/73] Update github actions --- .github/workflows/dockerimage.yml | 35 +++++++++++++++---------------- scripts/docker-slim.sh | 2 +- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 8ac1c63..0ae2f52 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -6,53 +6,52 @@ on: - 'v*' jobs: - main: + build: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Docker meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + uses: crazy-max/ghaction-docker-meta@v5 with: - images: antmoc/antmoc-ci - tag-semver: | - {{version}} - {{major}}.{{minor}} + images: | + antmoc/antmoc-ci + tags: | + type=semver,pattern={{version}} - name: Prepare arguments id: prep run: | - echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') - echo ::set-output name=vcs_url::$(git config --get remote.origin.url) - echo ::set-output name=vcs_ref::$(git rev-parse --short HEAD) + echo ::set-output name=oci_created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + echo ::set-output name=oci_source::$(git config --get remote.origin.url) + echo ::set-output name=oci_revision::$(git rev-parse --short HEAD) - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Build and push id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: . - file: ./Dockerfile platforms: linux/amd64 build-args: | - BUILD_DATE=${{ steps.prep.outputs.created }} - VCS_URL=${{ steps.prep.outputs.vcs_url }} - VCS_REF=${{ steps.prep.outputs.vcs_ref }} + OCI_CREATED=${{ steps.prep.outputs.oci_created }} + OCI_SOURCE=${{ steps.prep.outputs.oci_source }} + OCI_REVISION=${{ steps.prep.outputs.oci_revision }} push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/scripts/docker-slim.sh b/scripts/docker-slim.sh index 42488e0..655df32 100755 --- a/scripts/docker-slim.sh +++ b/scripts/docker-slim.sh @@ -1,7 +1,7 @@ # See https://github.com/slimtoolkit/slim IMAGE=${1:-"antmoc/antmoc-ci:0.1.16-alpha"} -NEWTAG=${IMAGE%-*}-slim +NEWTAG=${IMAGE%-*} /opt is the largest directory slim build \ From 8e20216b358ec350a670da8e23d537f3f3a11a1f Mon Sep 17 00:00:00 2001 From: one Date: Sat, 13 Apr 2024 17:23:53 +0800 Subject: [PATCH 72/73] Remove an unused argument --- Dockerfile | 2 +- Makefile | 3 --- README.md | 8 ++++---- scripts/docker-slim.sh | 1 - 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8a98a9c..cbf765f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ ARG SPACK_VERSION="0.21.2" ARG SPACK_IMAGE="spack/ubuntu-jammy" -FROM ${SPACK_IMAGE}:${SPACK_VERSION} AS builder +FROM ${SPACK_IMAGE}:${SPACK_VERSION} ARG UBUNTU_CODE ENV UBUNTU_CODE=${UBUNTU_CODE:-"jammy"} diff --git a/Makefile b/Makefile index 245b5c6..1b76a28 100644 --- a/Makefile +++ b/Makefile @@ -16,9 +16,6 @@ TARGET ?= x86_64 DOCKER_IMAGE ?= antmoc/antmoc-ci DOCKER_TAG := 0.1.16-alpha -# Default user -USER_NAME ?= hpcer - #=============================================================================== # Variables and objects #=============================================================================== diff --git a/README.md b/README.md index d2c7ee5..8090636 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ ANT-MOC CI Official ANT-MOC CI images starting from 0.1.15. -> Legacy images (before 0.1.15) could be found at -> -> - DockerHub: [leavesask/antmoc-ci](https://hub.docker.com/r/leavesask/antmoc-ci) -> - GitHub: [docker-antmoc-ci](https://github.com/alephpiece/docker-antmoc-ci) +Legacy images (before 0.1.15) could be found at + +- DockerHub: [leavesask/antmoc-ci](https://hub.docker.com/r/leavesask/antmoc-ci) +- GitHub: [docker-antmoc-ci](https://github.com/alephpiece/docker-antmoc-ci) diff --git a/scripts/docker-slim.sh b/scripts/docker-slim.sh index 655df32..c0815ae 100755 --- a/scripts/docker-slim.sh +++ b/scripts/docker-slim.sh @@ -3,7 +3,6 @@ IMAGE=${1:-"antmoc/antmoc-ci:0.1.16-alpha"} NEWTAG=${IMAGE%-*} -/opt is the largest directory slim build \ --mount $(pwd)/ant-moc:/opt/mnt/ant-moc \ --mount $(pwd)/scripts:/opt/mnt/scripts \ From 9d1a7e8b680771978619685a316341fbcccdf605 Mon Sep 17 00:00:00 2001 From: one Date: Wed, 1 May 2024 16:56:59 +0800 Subject: [PATCH 73/73] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1bf1d62..8a73720 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ __pycache__/ +/mirror /mirror/ /ant-moc/ /antmoc/