From f4ab2f2c94c76c272da8ca374b40265dd2756566 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Sun, 25 Aug 2024 22:07:23 +0200 Subject: [PATCH 1/4] Updated the Dockerfile templates for the weekly build of containers (#930) Updated and added more packages to the Dockerfile templates for the weekly build of containers. This should hopefully make the CI_build_wheels workflow should be much faster. Fixed also some bugs. --- .github/docker/Dockerfile-manylinux.template | 35 ++++++++++--- .../docker/Dockerfile-manylinux_x_y.template | 21 ++++++-- .github/docker/Dockerfile-musllinux.template | 51 +++++++++++++------ .github/docker/gen_dockerfile.sh | 2 +- .github/docker/run_cibuildwheel.sh | 4 ++ .github/workflows/container_builds_weekly.yml | 2 +- requirements_full.txt | 8 +-- 7 files changed, 90 insertions(+), 33 deletions(-) diff --git a/.github/docker/Dockerfile-manylinux.template b/.github/docker/Dockerfile-manylinux.template index f11ec5f8e..c90bb0376 100644 --- a/.github/docker/Dockerfile-manylinux.template +++ b/.github/docker/Dockerfile-manylinux.template @@ -37,6 +37,8 @@ FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest ARG PY_MINORS="8 9 10 11 12" +COPY requirements_full.txt requirements_dev.txt /tmp/ + {{ EXTRA_PRE }} # Enable rpmfusion for additional packages @@ -46,12 +48,13 @@ RUN \ https://mirrors.rpmfusion.org/free/el/rpmfusion-free-release-$(rpm --eval %{centos_ver}).noarch.rpm \ https://mirrors.rpmfusion.org/nonfree/el/rpmfusion-nonfree-release-$(rpm --eval %{centos_ver}).noarch.rpm && \ yum install -y \ + swig \ redland-devel \ rasqal-devel \ - swig \ libcurl-devel \ libxslt-devel \ - libxml2-devel && \ + libxml2-devel \ + libffi-devel && \ # hdf5-devel is needed for building the hdf5 plugin yum install -y hdf5-devel || true && \ # Unpack static libraries @@ -59,13 +62,33 @@ RUN \ # exist here. cd /opt/_internal && \ tar -Jxvf static-libs-for-embedding-only.tar.xz && \ + # Filter out challenging packages from requirements + sed \ + -e '/mongomock/d' \ + -i /tmp/requirements_dev.txt && \ + sed \ + -e '/pandas/d' \ + -e '/pyarrow/d' \ + -e '/tables/d' \ + -e '/openpyxl/d' \ + -e '/jinja2/d' \ + -e '/pydantic/d' \ + -e '/fortran-language-server/d' \ + -e '/matplotlib/d' \ + -e '/scikit-image/d' \ + -e '/ncempy/d' \ + -e '/h5py/d' \ + -i /tmp/requirements_full.txt && \ # Install required Python packages mkdir -p /ci/pip_cache && \ for minor in ${PY_MINORS}; do \ - python3.${minor} -m pip install -U pip && \ - python3.${minor} -m pip install -U setuptools wheel && \ - python3.${minor} -m pip install -U --cache-dir /ci/pip_cache \ - cmake oldest-supported-numpy; \ + python3.${minor} -m pip install --cache-dir=/ci/pip_cache -U pip && \ + python3.${minor} -m pip install --cache-dir=/ci/pip_cache \ + -U setuptools wheel && \ + python3.${minor} -m pip install --cache-dir=/ci/pip_cache \ + oldest-supported-numpy \ + -r /tmp/requirements_dev.txt \ + -r /tmp/requirements_full.txt; \ done && \ rm -rf /ci/pip_cache diff --git a/.github/docker/Dockerfile-manylinux_x_y.template b/.github/docker/Dockerfile-manylinux_x_y.template index df3a16c35..048a4a875 100644 --- a/.github/docker/Dockerfile-manylinux_x_y.template +++ b/.github/docker/Dockerfile-manylinux_x_y.template @@ -21,7 +21,7 @@ # # cd dlite # cd to DLite root directory # docker build -t dlite-manylinux \ -# -f .github/docker/Dockerfile-manylinux_2_24_x86_64 . +# -f .github/docker/dockerfiles/Dockerfile-manylinux_2_24_x86_64 . # # Run (for debugging): # @@ -39,6 +39,8 @@ ARG PY_MINORS="8 9 10 11 12" {{ EXTRA_PRE }} +COPY requirements_full.txt requirements_dev.txt /tmp/ + # Enable rpmfusion for additional packages RUN \ dnf update -y && \ @@ -58,13 +60,22 @@ RUN \ # exist here. cd /opt/_internal && \ tar -Jxvf static-libs-for-embedding-only.tar.xz && \ + # Filter out challenging packages from requirements_full.txt + #sed \ + # -e '/pyarrow/d' \ + # -e '/tables/d' \ + # -e '/scikit-image/d' \ + # -i /tmp/requirements_full.txt && \ # Install required Python packages mkdir -p /ci/pip_cache && \ for minor in ${PY_MINORS}; do \ - python3.${minor} -m pip install -U pip && \ - python3.${minor} -m pip install -U setuptools wheel && \ - python3.${minor} -m pip install -U --cache-dir /ci/pip_cache \ - cmake oldest-supported-numpy; \ + python3.${minor} -m pip install --cache-dir=/ci/pip_cache -U pip && \ + python3.${minor} -m pip install --cache-dir=/ci/pip_cache \ + -U setuptools wheel && \ + python3.${minor} -m pip install --cache-dir=/ci/pip_cache \ + oldest-supported-numpy \ + -r /tmp/requirements_dev.txt \ + -r /tmp/requirements_full.txt; \ done && \ rm -rf /ci/pip_cache diff --git a/.github/docker/Dockerfile-musllinux.template b/.github/docker/Dockerfile-musllinux.template index 7ca521c71..136a730c8 100644 --- a/.github/docker/Dockerfile-musllinux.template +++ b/.github/docker/Dockerfile-musllinux.template @@ -34,33 +34,52 @@ FROM quay.io/pypa/musllinux{{ TYPE }}_{{ ARCH }}:latest ARG PY_MINORS="8 9 10 11 12" +COPY requirements_full.txt requirements_dev.txt /tmp/ + +{{ EXTRA_PRE }} + # Do not use distutils distributed with setuptools # This is due to base changes in the distutils API, removing msvccompiler, # which is necessary for building the numpy wheel. ENV SETUPTOOLS_USE_DISTUTILS="stdlib" -RUN apk add -u \ - redland \ - rasqal \ - hdf5-dev \ - openssl-dev \ - swig && \ +RUN \ + apk add -u \ + redland-dev \ + rasqal-dev \ + hdf5-dev \ + openssl-dev \ + openblas-dev \ + swig && \ # Unpack static libraries # It's necessary to be in /opt/_internal because the internal libraries # exist here. cd /opt/_internal && \ tar -Jxvf static-libs-for-embedding-only.tar.xz && \ - # Filter minor versions - minors=$( for n in $PY_MINORS; do \ - if [ $(uname -m) == "i686" -o $n -lt 12 ]; then echo $n; fi; \ - done ) && \ - echo "*** Minors: $minors" && \ + # Filter out challenging packages from requirements_full.txt + sed \ + -e '/pandas/d' \ + -e '/pyarrow/d' \ + -e '/tables/d' \ + -e '/jinja2/d' \ + -e '/pydantic/d' \ + -e '/matplotlib/d' \ + -e '/scikit-image/d' \ + -e '/ncempy/d' \ + -e '/h5py/d' \ + -i /tmp/requirements_full.txt && \ + #-e 's/^pydantic.*/pydantic<2/' \ # Install required Python packages mkdir -p /ci/pip_cache && \ - for minor in ${minors}; do \ - python3.${minor} -m pip install -U pip && \ - python3.${minor} -m pip install -U setuptools wheel; \ - python3.${minor} -m pip install -U --cache-dir /ci/pip_cache \ - oldest-supported-numpy; \ + for minor in ${PY_MINORS}; do \ + python3.${minor} -m pip install --cache-dir=/ci/pip_cache -U pip && \ + python3.${minor} -m pip install --cache-dir=/ci/pip_cache \ + -U setuptools wheel && \ + python3.${minor} -m pip install --cache-dir=/ci/pip_cache \ + oldest-supported-numpy \ + -r /tmp/requirements_dev.txt \ + -r /tmp/requirements_full.txt; \ done && \ rm -rf /ci/pip_cache + +{{ EXTRA_POST }} diff --git a/.github/docker/gen_dockerfile.sh b/.github/docker/gen_dockerfile.sh index 7ffc374c9..54a470efe 100755 --- a/.github/docker/gen_dockerfile.sh +++ b/.github/docker/gen_dockerfile.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Write Dockerfile for given system to stdout. # # Usage: diff --git a/.github/docker/run_cibuildwheel.sh b/.github/docker/run_cibuildwheel.sh index e18b42a1f..5b0cd5e79 100755 --- a/.github/docker/run_cibuildwheel.sh +++ b/.github/docker/run_cibuildwheel.sh @@ -9,6 +9,10 @@ # and SYSTEM_TYPE is the specific linux system type, e.g., 2010 or 2014 for manylinux # and _1_1 or _1_2 for musllinux # More information: https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip +# +# Example: +# +# run_cibuildwheel.sh cp38-musllinux_x86_64 _1_1 set -eu if [[ $# -eq 0 ]]; then diff --git a/.github/workflows/container_builds_weekly.yml b/.github/workflows/container_builds_weekly.yml index 12d58f8e2..23dc892e7 100644 --- a/.github/workflows/container_builds_weekly.yml +++ b/.github/workflows/container_builds_weekly.yml @@ -95,7 +95,7 @@ jobs: if: env.BUILD_NEW_IMAGE == 'true' run: | mkdir -p docker_build_wheel - sh .github/docker/gen_dockerfile.sh ${{ matrix.system }} ${{ matrix.type }} ${{ matrix.arch }} \ + bash .github/docker/gen_dockerfile.sh ${{ matrix.system }} ${{ matrix.type }} ${{ matrix.arch }} \ > docker_build_wheel/Dockerfile-${{ matrix.system }}${{ matrix.type }}_${{ matrix.arch }} - name: Login to GitHub Container Registry diff --git a/requirements_full.txt b/requirements_full.txt index 2587481b9..e7e7ec5ed 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -25,10 +25,10 @@ typing_extensions>=4.1,<5 fortran-language-server>=1.12.0,<1.13 # For TEM demo -matplotlib>=3,<4 -scikit-image>=0.19,<1 -ncempy>=1.10,<2 -h5py>=3.9,<4 +##matplotlib>=3,<4 +##scikit-image>=0.19,<1 +##ncempy>=1.10,<2 +##h5py>=3.9,<4 #oteapi-dlite>=0.1.5,<1 #git+https://github.com/EMMC-ASBL/oteapi-dlite.git@fa1b820383eb54a1c37f32f7b8ac9406b556dace#egg=oteapi_dlite #otelib>=0.3.2,<1 From 83d0332730cbd306631e8ae0ce583e771a17d52b Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Mon, 26 Aug 2024 14:58:35 +0200 Subject: [PATCH 2/4] Update the documentation (#932) * Update the documentation about data models --- doc/user_guide/concepts.md | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/doc/user_guide/concepts.md b/doc/user_guide/concepts.md index f155f1b23..8d77f5185 100644 --- a/doc/user_guide/concepts.md +++ b/doc/user_guide/concepts.md @@ -93,17 +93,27 @@ As a user, you will almost always deal with only entities or data instances, but the hierarchy gives DLite a strong and theoretically well-defined schema. -The actual data or *Data instances* are instances of the *Entity* that -describes them. +An *instance* in DLite is a formal representation of a dataset. It is +called an instance, because it is an instance of a data model (or +*metadata* using the DLite terminology). +This is similar to instances of classes in object oriented programming. + +In DLite we often say that an instance is described by its metadata (aka data model). +The DLite metadata are themselves formalised and described by their meta-metadata. +Hence, metadata are instances, too. + +At the lowest level of abstraction we have the actual data or *data +instances*. +Metadata that describe *data instances* are (for historical reasons +and compatibility with SOFT) called *entities*. *Entities* are instances of the *EntitySchema*. -The *EntitySchema* is -an instance of the *BasicMetadataSchema* and an instance of itself -(meaning that it can be used to describe itself). -Hence, in DLite **everything is an instance**. -This has a practical implication that the API for instances can be applied -to all metadata as well. -Since the BasicMetadataSchema can describe itself, no more abstraction -levels are needed, making the DLite metadata schema complete and well-defined. +The *EntitySchema* is an instance of the *BasicMetadataSchema*, which is an +instance of itself (meaning that it can be used to describe itself). +Hence, in DLite **everything is an instance**. This has a practical +implication that the API for instances can be applied to all metadata +as well. Since the BasicMetadataSchema can describe itself, no more +abstraction levels are needed, making the DLite metadata schema +complete and well-defined. From Figure 2, one can also see that *[collections]* are simply instances of the *CollectionEntity*. From 8b2b329bc2aa4e1e8edc70be5d1ef2f078c9ca6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 05:33:45 +0000 Subject: [PATCH 3/4] Bump sphinx-autoapi from 3.2.1 to 3.3.1 (#938) Bumps [sphinx-autoapi](https://github.com/readthedocs/sphinx-autoapi) from 3.2.1 to 3.3.1. - [Release notes](https://github.com/readthedocs/sphinx-autoapi/releases) - [Changelog](https://github.com/readthedocs/sphinx-autoapi/blob/main/CHANGELOG.rst) - [Commits](https://github.com/readthedocs/sphinx-autoapi/compare/v3.2.1...v3.3.1) --- updated-dependencies: - dependency-name: sphinx-autoapi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_doc.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_doc.txt b/requirements_doc.txt index 15971ff17..de8bd2a82 100644 --- a/requirements_doc.txt +++ b/requirements_doc.txt @@ -3,7 +3,7 @@ beautifulsoup4==4.12.3 lxml==5.3.0 Sphinx==8.0.2 -sphinx-autoapi==3.2.1 +sphinx-autoapi==3.3.1 sphinx-autobuild==2024.4.16 sphinx-book-theme==1.1.3 sphinx-design==0.6.1 From 4fffa98c4e5724226b3f0dd173cefcd90a96f8dd Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 3 Sep 2024 22:08:41 +0200 Subject: [PATCH 4/4] Require rdflib for test_dataset*.py (#936) --- bindings/python/tests/test_dataset1_save.py | 1 + bindings/python/tests/test_dataset2_load.py | 1 + 2 files changed, 2 insertions(+) diff --git a/bindings/python/tests/test_dataset1_save.py b/bindings/python/tests/test_dataset1_save.py index fedc2cddc..5007b8c30 100644 --- a/bindings/python/tests/test_dataset1_save.py +++ b/bindings/python/tests/test_dataset1_save.py @@ -3,6 +3,7 @@ import dlite from dlite.testutils import raises, importskip importskip("tripper") +importskip("rdflib") from tripper import DCTERMS, MAP, OWL, RDF, RDFS, XSD, Triplestore from tripper.utils import en diff --git a/bindings/python/tests/test_dataset2_load.py b/bindings/python/tests/test_dataset2_load.py index 7211e0111..04ca0c0c7 100644 --- a/bindings/python/tests/test_dataset2_load.py +++ b/bindings/python/tests/test_dataset2_load.py @@ -3,6 +3,7 @@ import dlite from dlite.testutils import raises, importskip importskip("tripper") +importskip("rdflib") from tripper import MAP, Triplestore