Skip to content

Commit

Permalink
Add pip packaging for ADIOS2
Browse files Browse the repository at this point in the history
  • Loading branch information
scottwittenburg committed Dec 15, 2023
1 parent d192f27 commit 0272d85
Show file tree
Hide file tree
Showing 14 changed files with 344 additions and 33 deletions.
100 changes: 100 additions & 0 deletions .github/workflows/pypackaging.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
name: Python Packaging

on:
workflow_dispatch:
inputs:
overrideVersion:
description: Manually force a version
pull_request:
push:
branches:
- master
- release_[0-9]+
tags:
- v[0-9]+.[0-9]+.[0-9]+*
release:
types:
- published

env:
ADIOS2_CUSTOM_VERSION_OVERRIDE: ${{ github.event.inputs.overrideVersion }}

jobs:
make_sdist:
name: Make SDist
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Generate common version file
run: cmake -P scripts/ci/gh-actions/config/adios-version.cmake && echo VERSION.TXT

- name: Build SDist
run: pipx run build --sdist

- uses: actions/upload-artifact@v3
with:
path: dist/*.tar.gz

build_wheels:
name: Wheel on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
# os: [ubuntu-latest, windows-latest, macos-latest]
os: [ubuntu-latest]

steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Generate common version file
run: cmake -P scripts/ci/gh-actions/config/adios-version.cmake && echo VERSION.TXT

- uses: pypa/cibuildwheel@v2.16
env:
CIBW_BUILD: cp*-manylinux_x86_64

- name: Upload wheels
uses: actions/upload-artifact@v3
with:
path: wheelhouse/*.whl

upload_pypi:
needs: [build_wheels, make_sdist]
environment: pypi
permissions:
id-token: write
runs-on: ubuntu-latest
if: github.event_name == 'release' && github.event.action == 'published'
steps:
- uses: actions/download-artifact@v3
with:
name: artifact
path: dist

- name: Publish package distributions to PyPI
uses: pypa/gh-action-pypi-publish@release/v1

upload_test_pypi:
needs: [build_wheels, make_sdist]
environment: testpypi
permissions:
id-token: write
runs-on: ubuntu-latest
# Upload to Test PyPI for every commit on main branch
if: github.event_name == 'push' && github.event.ref == 'refs/heads/master'
steps:
- uses: actions/download-artifact@v3
with:
name: artifact
path: dist

- name: Publish package distributions to TestPyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
repository-url: https://test.pypi.org/legacy/
25 changes: 20 additions & 5 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,19 @@ project(ADIOS2 VERSION ${ADIOS2_VERSION})
# Some boilerplate to setup nice output directories
#------------------------------------------------------------------------------#
include(GNUInstallDirs)

if (ADIOS2_USE_PIP)
# Only UNIX is supported currently
if (UNIX)
# Linux bundles what libraries we have when they're put beside the modules.
set(CMAKE_INSTALL_LIBDIR "adios2")
set(CMAKE_INSTALL_INCLUDEDIR "adios2/include")
# ELF loader settings.
set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)
list(APPEND CMAKE_INSTALL_RPATH "$ORIGIN")
endif ()
endif ()

set(CMAKE_INSTALL_CMAKEDIR ${CMAKE_INSTALL_LIBDIR}/cmake/adios2
CACHE STRING "Installation CMake subdirectory")
mark_as_advanced(CMAKE_INSTALL_CMAKEDIR)
Expand Down Expand Up @@ -172,6 +185,8 @@ adios_option(Sodium "Enable support for Sodium for encryption" AUTO)
adios_option(Catalyst "Enable support for in situ visualization plugin using ParaView Catalyst" AUTO)
adios_option(AWSSDK "Enable support for S3 compatible storage using AWS SDK's S3 module" OFF)
adios_option(Derived_Variable "Enable support for derived variables" OFF)
adios_option(PIP "Enable support for pip packaging" OFF)
mark_as_advanced(ADIOS2_USE_PIP)
include(${PROJECT_SOURCE_DIR}/cmake/DetectOptions.cmake)

if(ADIOS2_HAVE_CUDA OR ADIOS2_HAVE_Kokkos_CUDA)
Expand Down Expand Up @@ -242,7 +257,7 @@ endif()


set(ADIOS2_CONFIG_OPTS
DataMan DataSpaces HDF5 HDF5_VOL MHS SST Fortran MPI Python Blosc2 BZip2
DataMan DataSpaces HDF5 HDF5_VOL MHS SST Fortran MPI Python PIP Blosc2 BZip2
LIBPRESSIO MGARD MGARD_MDR PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem UCX
ZeroMQ Profiling Endian_Reverse Derived_Variable AWSSDK GPU_Support CUDA Kokkos
Kokkos_CUDA Kokkos_HIP Kokkos_SYCL
Expand Down Expand Up @@ -283,12 +298,12 @@ if(BUILD_SHARED_LIBS AND ADIOS2_RUN_INSTALL_TEST)
set(ADIOS2_RUN_INSTALL_TEST FALSE)
endif()
else()
set(CMAKE_INSTALL_RPATH "@loader_path/${relative_base}/${CMAKE_INSTALL_LIBDIR}")
list(APPEND CMAKE_INSTALL_RPATH "@loader_path/${relative_base}/${CMAKE_INSTALL_LIBDIR}")
endif()

elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
# Linux needs some specialized RPATH handling
set(CMAKE_INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}")
list(APPEND CMAKE_INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}")
endif()
endif()

Expand Down Expand Up @@ -371,8 +386,8 @@ endif()
message("")
message("ADIOS2 build configuration:")
message(" ADIOS Version: ${ADIOS2_VERSION}")
message(" C++ Compiler : ${CMAKE_CXX_COMPILER_ID} "
"${CMAKE_CXX_COMPILER_VERSION} "
message(" C++ Compiler : ${CMAKE_CXX_COMPILER_ID} "
"${CMAKE_CXX_COMPILER_VERSION} "
"${CMAKE_CXX_COMPILER_WRAPPER}")
message(" ${CMAKE_CXX_COMPILER}")
message("")
Expand Down
9 changes: 5 additions & 4 deletions ReadMe.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
[![GitHub (pre-)release](https://img.shields.io/github/release/ornladios/adios2/all.svg)]()
[![Spack Version](https://img.shields.io/spack/v/adios2.svg)](https://spack.readthedocs.io/en/latest/package_list.html#adios2)
[![Conda Version](https://img.shields.io/conda/vn/conda-forge/adios2)](https://anaconda.org/conda-forge/adios2)
[![PyPI version](https://badge.fury.io/py/adios2.svg)](https://badge.fury.io/py/adios2)

[![Circle CI](https://circleci.com/gh/ornladios/ADIOS2.svg?style=shield)](https://circleci.com/gh/ornladios/ADIOS2)

Expand All @@ -23,7 +24,7 @@ ADIOS2 can be used on supercomputers, cloud systems, and personal computers.
ADIOS2 focuses on:

1. **Performance** I/O scalability in high performance computing (HPC) applications.
2. **Adaptability** unified interfaces to allow for several modes of transport (files, memory-to-memory)
2. **Adaptability** unified interfaces to allow for several modes of transport (files, memory-to-memory)
3. **Ease of Use** two-level application programming interface (APIs)
* Full APIs for HPC applications: C++11, Fortran 90, C 99, Python 2 and 3
* Simplified High-Level APIs for data analysis: Python 2 and 3, C++11, Matlab
Expand Down Expand Up @@ -63,7 +64,7 @@ For a `cmake` configuration example see [scripts/runconf/runconf.sh](https://git
* Docker images: under [scripts/docker](https://github.com/ornladios/ADIOS2/tree/master/scripts/docker)


Once ADIOS2 is installed refer to:
Once ADIOS2 is installed refer to:

* [Linking ADIOS2](https://adios2.readthedocs.io/en/latest/setting_up/setting_up.html#linking-adios-2)

Expand All @@ -78,7 +79,7 @@ Once ADIOS2 is installed refer to:

ADIOS2 is an open source project: Questions, discussion, and contributions are welcome. Join us at:

- Mailing list: adios-ecp@kitware.com
- Mailing list: adios-ecp@kitware.com
- Github Discussions: https://github.com/ornladios/ADIOS2/discussions

## Reporting Bugs
Expand All @@ -103,7 +104,7 @@ See the accompanying [Copyright.txt](Copyright.txt) for more details.

* scripts - Project maintenance and development scripts

* source - Internal source code for private components
* source - Internal source code for private components
* adios2 - source directory for the ADIOS2 library to be installed under install-dir/lib/libadios2.
* utils - source directory for the binary utilities, to be installed under install-dir/bin

Expand Down
21 changes: 16 additions & 5 deletions bindings/Python/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -49,16 +49,27 @@ set_target_properties(adios2_py PROPERTIES

string(REGEX REPLACE "[^/]+" ".." relative_base "${CMAKE_INSTALL_PYTHONDIR}/adios2")
if(CMAKE_SYSTEM_NAME MATCHES "Linux")
set_target_properties(adios2_py PROPERTIES
INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}"
)
if (NOT ADIOS2_USE_PIP)
set_target_properties(adios2_py PROPERTIES
INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}"
)
endif()
endif()

set(install_location ${CMAKE_INSTALL_PYTHONDIR})
if (ADIOS2_USE_PIP)
set(install_location ${CMAKE_INSTALL_LIBDIR})
endif()

install(TARGETS adios2_py
DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2
DESTINATION ${install_location}
COMPONENT adios2_python-python
)
install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py
DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2
DESTINATION ${install_location}
COMPONENT adios2_python-python
)
install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test
DESTINATION ${install_location}
COMPONENT adios2_python-python
)
Empty file.
46 changes: 46 additions & 0 deletions bindings/Python/test/simple_read_write.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import adios2

import sys
import unittest


DATA_FILENAME = "hello-world-py.bp"

class TestSimpleReadWrite(unittest.TestCase):

def _write(self, ad, greeting):
"""write a string to a bp file"""
io = ad.DeclareIO("hello-world-writer")
var_greeting = io.DefineVariable("Greeting")
w = io.Open(DATA_FILENAME, adios2.Mode.Write)
w.BeginStep()
w.Put(var_greeting, greeting)
w.EndStep()
w.Close()
return 0

def _read(self, ad):
"""read a string from to a bp file"""
io = ad.DeclareIO("hello-world-reader")
r = io.Open(DATA_FILENAME, adios2.Mode.Read)
r.BeginStep()
var_greeting = io.InquireVariable("Greeting")
message = r.Get(var_greeting)
r.EndStep()
r.Close()
return message

def test_simple_read_write(self):
"""driver function"""
print("ADIOS2 version {0}".format(adios2.__version__))
ad = adios2.ADIOS()
greeting = "Hello World from ADIOS2"
self._write(ad, greeting)
message = self._read(ad)
print("{}".format(message))
self.assertEqual(greeting, message)
return 0


if __name__ == '__main__':
unittest.main()
13 changes: 9 additions & 4 deletions cmake/ADIOSFunctions.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,19 @@ function(setup_version BASE)
if(res EQUAL 0 AND out MATCHES "[^-]*-([^-]*)-g([a-f0-9]*)")
set(ver_tweak ${CMAKE_MATCH_1})
set(ver_gitsha ${CMAKE_MATCH_2})
endif()
endif()
endif()
endif()

if(ver_tweak)
set(ADIOS2_VERSION ${BASE}.${ver_tweak} PARENT_SCOPE)
if(ADIOS2_USE_PIP AND EXISTS "${CMAKE_SOURCE_DIR}/VERSION.TXT")
file(READ "VERSION.TXT" version_from_file)
set(ADIOS2_VERSION ${version_from_file} PARENT_SCOPE)
else()
set(ADIOS2_VERSION ${BASE} PARENT_SCOPE)
if(ver_tweak)
set(ADIOS2_VERSION ${BASE}.${ver_tweak} PARENT_SCOPE)
else()
set(ADIOS2_VERSION ${BASE} PARENT_SCOPE)
endif()
endif()

if(ver_gitsha)
Expand Down
8 changes: 7 additions & 1 deletion cmake/DetectOptions.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -399,7 +399,9 @@ if(NOT SHARED_LIBS_SUPPORTED)
endif()
endif()

if(ADIOS2_USE_Python STREQUAL AUTO)
if(ADIOS2_USE_PIP)
find_package(Python 3 REQUIRED COMPONENTS Interpreter Development.Module NumPy)
elseif(ADIOS2_USE_Python STREQUAL AUTO)
find_package(Python 3 COMPONENTS Interpreter Development NumPy)
if(Python_FOUND AND ADIOS2_HAVE_MPI)
find_package(PythonModule COMPONENTS mpi4py mpi4py/mpi4py.h)
Expand Down Expand Up @@ -543,6 +545,10 @@ if(ADIOS2_USE_Endian_Reverse STREQUAL ON)
set(ADIOS2_HAVE_Endian_Reverse TRUE)
endif()

if(ADIOS2_USE_PIP STREQUAL ON)
set(ADIOS2_HAVE_PIP TRUE)
endif()

# Sodium for EncryptionOperator
if(ADIOS2_USE_Sodium STREQUAL AUTO)
find_package(Sodium)
Expand Down
Loading

0 comments on commit 0272d85

Please sign in to comment.