diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..6fddca0d6 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/keyvi.yml b/.github/workflows/keyvi.yml index d55b09254..aa13a8aca 100644 --- a/.github/workflows/keyvi.yml +++ b/.github/workflows/keyvi.yml @@ -36,7 +36,7 @@ jobs: uses: actions/checkout@v2 - name: ccache - uses: hendrikmuhs/ccache-action@v1.1 + uses: hendrikmuhs/ccache-action@v1.2.11 with: key: ${{ matrix.os }}-${{ matrix.type }} diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml deleted file mode 100644 index 3b65fe1d5..000000000 --- a/.github/workflows/manylinux.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Manylinux - -on: - push: - pull_request: - branches: [ master ] - workflow_dispatch: - -jobs: - manylinux: - strategy: - matrix: - python_path: [ cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311 ] - manylinux_image: [ keyvidev/manylinux-builder-x86_64, keyvidev/manylinux-builder-aarch64 ] - - runs-on: ubuntu-latest - steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - uses: actions/checkout@v2 - - name: Build manylinux package - uses: addnab/docker-run-action@v3 - with: - image: ${{ matrix.manylinux_image }} - options: -v ${{ github.workspace }}:/repo -e PYTHON_PATH=${{ matrix.python_path }} - - run: | - cd /repo/python - - PYBIN=/opt/python/${PYTHON_PATH}/bin - - ${PYBIN}/pip install -r requirements.txt - - # Build - ${PYBIN}/python setup.py bdist_wheel -d dist - - # Bundle external shared libraries into the wheels - for wheel in dist/*.whl; do - auditwheel repair ${wheel} -w wheelhouse/ - done - - # Install and test - ${PYBIN}/pip install keyvi --no-index -f wheelhouse/ - - cd ~ - ${PYBIN}/py.test /repo/python/tests/ - ${PYBIN}/py.test /repo/python/integration-tests/ - - - - name: Publish package - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - uses: addnab/docker-run-action@v3 - with: - image: ${{ matrix.manylinux_image }} - options: -v ${{ github.workspace }}:/repo -e PYTHON_PATH=${{ matrix.python_path }} -e PYPI_PASSWORD=${{ secrets.pypi_password }} - run: | - PYBIN=/opt/python/${PYTHON_PATH}/bin - ${PYBIN}/pip install twine - ${PYBIN}/python -m twine upload -u __token__ -p "${PYPI_PASSWORD}" /repo/python/wheelhouse/* diff --git a/.github/workflows/python-cibuildwheel.yml b/.github/workflows/python-cibuildwheel.yml new file mode 100644 index 000000000..77afe5b92 --- /dev/null +++ b/.github/workflows/python-cibuildwheel.yml @@ -0,0 +1,183 @@ +name: Python cibuildwheel + +on: + push: + pull_request: + branches: [ master ] + workflow_dispatch: + +jobs: + build_wheels: + name: cibuildwheel ${{ matrix.os }}/${{ matrix.arch }}/${{ matrix.flavor }}/${{ matrix.target }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-22.04, macos-12] + # separate archs, so they use individual caches + arch: [ 'x86_64', 'arm64' ] + flavor: ['cpython', 'pypy'] + # separate musl and many on linux, for mac we just skip one of those + target: [ 'many', 'musl' ] + exclude: + - os: macos-12 + target: musl + - os: ubuntu-22.04 + target: musl + flavor: pypy + - os: macos-12 + arch: arm64 + flavor: pypy + steps: + - uses: actions/checkout@v4 + - name: Set up QEMU + if: ${{ (runner.os == 'Linux') && (matrix.arch == 'arm64') }} + uses: docker/setup-qemu-action@v3 + with: + platforms: all + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2.11 + with: + key: ${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.target }}-${{ matrix.flavor }}-python + + - name: Sets env for x86_64 + run: | + echo "CIBW_ARCHS_LINUX=auto64" >> $GITHUB_ENV + echo "CIBW_ARCHS_MACOS=x86_64" >> $GITHUB_ENV + if: matrix.arch == 'x86_64' + + - name: Sets env for arm64 + run: | + echo "CIBW_ARCHS_LINUX=aarch64" >> $GITHUB_ENV + echo "CIBW_ARCHS_MACOS=arm64" >> $GITHUB_ENV + if: matrix.arch == 'arm64' + + - name: Skip manylinux for musllinux target + if: ${{ (runner.os == 'Linux') && (matrix.target == 'musl') }} + run: | + echo "CIBW_SKIP=*manylinux*" >> $GITHUB_ENV + + - name: Skip musllinux for manylinux target + if: ${{ (runner.os == 'Linux') && (matrix.target == 'many') }} + run: | + echo "CIBW_SKIP=*musllinux*" >> $GITHUB_ENV + + - name: Skip pypy for cpython + if: ${{ matrix.flavor == 'cpython' }} + run: | + echo "CIBW_SKIP=${{ env.CIBW_SKIP }} pp*" >> $GITHUB_ENV + + - name: Skip cpython for pypy + if: ${{ matrix.flavor == 'pypy' }} + run: | + echo "CIBW_SKIP=${{ env.CIBW_SKIP }} cp*" >> $GITHUB_ENV + + - name: install mac dependencies + if: ${{ runner.os == 'macOS' }} + run: | + brew update && \ + brew install ccache + + - name: install mac dependencies X86_64 + if: ${{ (runner.os == 'macOS') && (matrix.arch == 'x86_64') }} + run: | + brew update && \ + brew install zlib snappy boost + + - name: install mac dependencies arm64 + if: ${{ (runner.os == 'macOS') && (matrix.arch == 'arm64') }} + run: | + set -e + echo "MACOSX_DEPLOYMENT_TARGET=12.3.0" >> $GITHUB_ENV + echo "_CMAKE_PREFIX_PATH=${{ github.workspace }}/arm64-homebrew" >> $GITHUB_ENV + echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=DYLD_LIBRARY_PATH=${{ github.workspace }}/arm64-homebrew delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}" >> $GITHUB_ENV + mkdir arm64-homebrew && curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C arm64-homebrew + PACKAGES=( icu4c xz lz4 zstd zlib snappy boost ) + for PACKAGE in "${PACKAGES[@]}" + do + response=$(arm64-homebrew/bin/brew fetch --force --bottle-tag=arm64_monterey $PACKAGE | grep Downloaded ) + download_path=$(echo $response | xargs -n 1 | tail -1) + arm64-homebrew/bin/brew reinstall -vd $download_path + done + arm64-homebrew/bin/brew config + ls /Users/runner/work/keyvi/keyvi/arm64-homebrew + + - name: Build python wheels for ${{ matrix.os }} on ${{ matrix.arch }} + uses: pypa/cibuildwheel@v2.16.2 + env: + # Skip CPython 3.6 and CPython 3.7 + CIBW_SKIP: ${{ env.CIBW_SKIP }} cp36-* cp37-* + + # skip testing all python versions on linux arm, only test 3.12 + # skip tests on pypy, currently fails for indexer tests + CIBW_TEST_SKIP: "*p{38,39,310,311}-m*linux_aarch64 pp*" + + # (many)linux custom docker images + CIBW_MANYLINUX_X86_64_IMAGE: 'keyvidev/manylinux-builder-x86_64' + CIBW_MANYLINUX_AARCH64_IMAGE: 'keyvidev/manylinux-builder-aarch64' + CIBW_MUSLLINUX_X86_64_IMAGE: 'keyvidev/musllinux-builder-x86_64' + CIBW_MUSLLINUX_AARCH64_IMAGE: 'keyvidev/musllinux-builder-aarch64' + + # ccache using path + CIBW_ENVIRONMENT_MACOS: PATH=/usr/local/opt/ccache/libexec:$PATH + CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/bin:/usr/lib/ccache:$PATH CCACHE_DIR=/host${{ github.workspace }}/.ccache CCACHE_CONFIGPATH=/host/home/runner/.config/ccache/ccache.conf + + # python dependencies + CIBW_BEFORE_BUILD: pip install -r python/requirements.txt + + # testing + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: > + python -m pytest {package}/tests && + python -m pytest {package}/integration-tests + + # for debugging set this to 1,2 or 3 + # CIBW_BUILD_VERBOSITY: 2 + with: + package-dir: python + + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + + build_sdist: + name: sdist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: install Linux deps + run: | + sudo apt-get update && \ + sudo apt-get install -y libsnappy-dev libzzip-dev zlib1g-dev libboost-all-dev ccache + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2.11 + with: + key: ${{ matrix.os }}-sdist-python + + - name: Build SDist + run: | + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + cd python && \ + python -m pip install -r requirements.txt && \ + python setup.py sdist -d wheelhouse && \ + python -m pip uninstall -y autowrap && \ + python -m pip install wheelhouse/*.tar.gz -v && \ + python -m pytest tests && \ + python -m pip uninstall -y keyvi + + - uses: actions/upload-artifact@v3 + with: + path: python/wheelhouse/*.tar.gz + + upload_all: + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + if: github.event_name == 'release' && github.event.action == 'published' + steps: + - uses: actions/download-artifact@v4 + with: + name: artifact + path: dist + + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml deleted file mode 100644 index 1d6238c48..000000000 --- a/.github/workflows/python.yml +++ /dev/null @@ -1,116 +0,0 @@ -# Build python bindings -name: Build python bindings - -# Controls when the action will run. -on: - # Triggers the workflow on push or pull request events but only for the master branch - push: - pull_request: - branches: [ master ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - build: - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12' ] - os: [ 'macos-latest', 'ubuntu-22.04' ] - include: - # test sdist on ubuntu only - - os: ubuntu-22.04 - python-version: '3.11' - sdist: true - build_direct: true - name: Python ${{ matrix.python-version }} on ${{ matrix.os }} build - steps: - - name: install Linux deps - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y libsnappy-dev libzzip-dev zlib1g-dev libboost-all-dev - sudo apt-get install ccache - - name: install macOS deps - if: runner.os == 'macOS' - run: | - brew update - brew install zlib snappy boost - brew install ccache - - name: checkout from git - uses: actions/checkout@v2 - - - name: ccache - uses: hendrikmuhs/ccache-action@v1.1 - with: - key: ${{ matrix.os }}-python - - - name: Setup python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - architecture: x64 - - - name: Install python dependencies - run: | - export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" - python -m pip install -r python/requirements.txt - - - name: build and test python extension - if: matrix.build_direct - run: | - export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" - cd python - python setup.py build --mode Release - python setup.py install --user - python -m pytest tests - python -m pytest integration-tests - python -m pip uninstall -y keyvi - - - name: bdist - run: | - export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" - cd python - python setup.py bdist_wheel -d wheelhouse - - - name: delocate - if: runner.os == 'macOS' - run: | - cd python - delocate-listdeps wheelhouse/keyvi*.whl - delocate-wheel wheelhouse/keyvi*.whl - delocate-listdeps wheelhouse/keyvi*.whl - - - name: remove macos deps - if: runner.os == 'macOS' - run: | - brew remove zlib - brew remove --ignore-dependencies snappy - - - name: bdist test - run: | - cd python - python -m pip install --user wheelhouse/*.whl - python -m pytest tests - python -m pip uninstall -y keyvi - - - name: sdist - if: matrix.sdist - run: | - export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" - cd python - rm -rf wheelhouse - python setup.py sdist -d wheelhouse - python -m pip uninstall -y autowrap - python -m pip install wheelhouse/*.tar.gz -v - python -m pytest tests - python -m pip uninstall -y keyvi - - - name: Publish package - if: ( matrix.sdist || runner.os == 'macOS' ) && github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - env: - PYPI_PASSWORD: ${{ secrets.pypi_password }} - run: | - python -m twine upload -u __token__ -p "${PYPI_PASSWORD}" python/wheelhouse/* diff --git a/CMakeLists.txt b/CMakeLists.txt index a8cdfec16..d61786661 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,9 +1,6 @@ cmake_minimum_required(VERSION 3.9) project(keyvi) -EXECUTE_PROCESS( COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE ARCHITECTURE ) -message( STATUS "Architecture: ${ARCHITECTURE}" ) - #### Build Type if (CMAKE_BUILD_TYPE) string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER) @@ -22,9 +19,9 @@ set(CMAKE_CXX_EXTENSIONS OFF) set (_KEVYI_COMPILE_OPTIONS "-Wall") set (_KEYVI_COMPILE_DEFINITIONS "RAPIDJSON_HAS_STDSTRING") -if(${ARCHITECTURE} STREQUAL "x86_64") +if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64") set (_KEYVI_CXX_FLAGS "-msse4.2") - message( STATUS "Architecture: ${ARCHITECTURE} detected: added -msse4.2 compile flag") + message( STATUS "Architecture: ${CMAKE_SYSTEM_PROCESSOR} detected: added -msse4.2 compile flag") endif() set (_OS_LIBRARIES "") diff --git a/python/requirements.txt b/python/requirements.txt index 39b0abbec..b2af40ec1 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,7 +1,4 @@ autowrap>=0.16.0 -delocate>=0.10.2 msgpack>=1.0.0 pytest>=7.1.1 -twine>=4.0.0 -wheel>=0.37.1 -cython>=3.0 \ No newline at end of file +cython>=3.0 diff --git a/python/setup.py b/python/setup.py index eef34aa55..1781f0ec2 100644 --- a/python/setup.py +++ b/python/setup.py @@ -17,13 +17,13 @@ from contextlib import contextmanager from os import path -pykeyvi_pyx = '_core.pyx' -pykeyvi_cpp = '_core.cpp' -pykeyvi_p_cpp = '_core_p.cpp' -keyvi_cpp_source = '../keyvi' -keyvi_cpp = 'src/cpp' -keyvi_cpp_link = path.join(keyvi_cpp, 'keyvi') -keyvi_build_dir = path.join(keyvi_cpp, 'build-{}'.format(platform.platform())) +pykeyvi_pyx = "_core.pyx" +pykeyvi_cpp = "_core.cpp" +pykeyvi_p_cpp = "_core_p.cpp" +keyvi_cpp_source = "../keyvi" +keyvi_cpp = "src/cpp" +keyvi_cpp_link = path.join(keyvi_cpp, "keyvi") +keyvi_build_dir = path.join(keyvi_cpp, "build-{}".format(platform.platform())) here = os.path.abspath(os.path.dirname(__file__)) try: @@ -41,7 +41,7 @@ VERSION = "{}.{}.{}".format(VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) if not IS_RELEASED: - VERSION += '.dev{}'.format(VERSION_DEV) + VERSION += ".dev{}".format(VERSION_DEV) ################### @@ -53,13 +53,14 @@ def wrapper(*args, **kwargs): wrapper.has_run = True wrapper.ret = f(*args, **kwargs) return wrapper.ret + wrapper.has_run = False wrapper.ret = None return wrapper def write_version_file(): - version_file_path = os.path.join(here, 'src/py/keyvi/_version.py') + version_file_path = os.path.join(here, "src/py/keyvi/_version.py") content = """ # THIS FILE IS GENERATED FROM KEYVI SETUP.PY @@ -67,23 +68,26 @@ def write_version_file(): """.format(VERSION) - with open(version_file_path, 'w') as f_out: + with open(version_file_path, "w") as f_out: f_out.write(content) + def clean_pykeyvi_build_directory(): if os.path.exists(keyvi_build_dir): remove_tree(keyvi_build_dir) + def generate_pykeyvi_source(): - addons = glob.glob('src/addons/*') - pxds = glob.glob('src/pxds/*') - converters = 'src/converters' - converter_files = glob.glob(path.join(converters, '*')) + addons = glob.glob("src/addons/*") + pxds = glob.glob("src/pxds/*") + converters = "src/converters" + converter_files = glob.glob(path.join(converters, "*")) max_modification_time = max([path.getmtime(fn) for fn in addons + pxds + converter_files]) if not path.exists(pykeyvi_cpp) or max_modification_time > path.getmtime(pykeyvi_cpp): try: import autowrap.Main + autowrap.Main.run(pxds, addons, [converters], pykeyvi_pyx) # rewrite generated cpp to use std::shared_ptr instead of boost::shared_ptr with open(pykeyvi_cpp, "rt") as fin: @@ -91,13 +95,14 @@ def generate_pykeyvi_source(): for line in fin: if line.find("shared_ptr.hpp") > 0: continue - fout.write(line.replace('boost::shared_ptr', 'std::shared_ptr')) + fout.write(line.replace("boost::shared_ptr", "std::shared_ptr")) except: if not path.exists(pykeyvi_cpp): raise else: - print ("Could not find autowrap, probably running from sdist environment") + print("Could not find autowrap, probably running from sdist environment") + @contextmanager def symlink_keyvi(): @@ -106,35 +111,53 @@ def symlink_keyvi(): if not path.exists(keyvi_cpp): os.makedirs(keyvi_cpp) os.symlink(path.abspath(keyvi_cpp_source), keyvi_cpp_link) - shutil.copy('../CMakeLists.txt', path.join(keyvi_cpp, 'CMakeLists.txt')) - shutil.copytree('../cmake_modules', path.join(keyvi_cpp, 'cmake_modules')) + shutil.copy("../CMakeLists.txt", path.join(keyvi_cpp, "CMakeLists.txt")) + shutil.copytree("../cmake_modules", path.join(keyvi_cpp, "cmake_modules")) keyvi_source_path = os.path.realpath(os.path.join(os.getcwd(), keyvi_cpp_source)) pykeyvi_source_path = os.path.join(os.getcwd(), keyvi_cpp_link) yield (pykeyvi_source_path, keyvi_source_path) finally: os.unlink(keyvi_cpp_link) - os.remove(path.join(keyvi_cpp, 'CMakeLists.txt')) - shutil.rmtree(path.join(keyvi_cpp, 'cmake_modules')) + os.remove(path.join(keyvi_cpp, "CMakeLists.txt")) + shutil.rmtree(path.join(keyvi_cpp, "cmake_modules")) else: yield None, None + @run_once -def cmake_configure(build_path, build_type, zlib_root, additional_compile_flags): +def cmake_configure( + build_path, + build_type, + zlib_root, + additional_compile_flags, + osx_architectures=None, + cmake_prefix_path=None, +): # needed for shared library - CMAKE_CXX_FLAGS = additional_compile_flags + ' -fPIC' + CMAKE_CXX_FLAGS = additional_compile_flags + " -fPIC" - cmake_configure_cmd = 'mkdir -p {}'.format(build_path) - cmake_configure_cmd += ' && cd {}'.format(build_path) - cmake_configure_cmd += ' && cmake' \ - ' -D CMAKE_CXX_FLAGS="{CXX_FLAGS}"'.format(CXX_FLAGS=CMAKE_CXX_FLAGS.strip()) - cmake_configure_cmd += ' -D CMAKE_BUILD_TYPE={BUILD_TYPE}'.format(BUILD_TYPE=build_type) + cmake_configure_cmd = "mkdir -p {}".format(build_path) + cmake_configure_cmd += " && cd {}".format(build_path) + cmake_configure_cmd += " && cmake" ' -D CMAKE_CXX_FLAGS="{CXX_FLAGS}"'.format( + CXX_FLAGS=CMAKE_CXX_FLAGS.strip() + ) + cmake_configure_cmd += f" -D CMAKE_BUILD_TYPE={build_type}" - if zlib_root is not None: - cmake_configure_cmd += ' -D ZLIB_ROOT={ZLIB_ROOT}'.format(ZLIB_ROOT=zlib_root) - cmake_configure_cmd += ' ..' + if osx_architectures is not None: + cmake_configure_cmd += ' -D CMAKE_OSX_ARCHITECTURES="{OSX_ARCH}"'.format( + OSX_ARCH=osx_architectures + ) - print ("Building in {0} mode".format(build_type)) - print ("Run keyvi C++ cmake: " + cmake_configure_cmd) + if zlib_root is not None: + cmake_configure_cmd += " -D ZLIB_ROOT={ZLIB_ROOT}".format(ZLIB_ROOT=zlib_root) + if cmake_prefix_path is not None: + cmake_configure_cmd += " -D CMAKE_PREFIX_PATH={CMAKE_PREFIX_PATH}".format( + CMAKE_PREFIX_PATH=cmake_prefix_path + ) + cmake_configure_cmd += " .." + + print("Building in {0} mode".format(build_type)) + print("Run keyvi C++ cmake: " + cmake_configure_cmd) subprocess.call(cmake_configure_cmd, shell=True) cmake_flags = {} @@ -144,50 +167,50 @@ def cmake_configure(build_path, build_type, zlib_root, additional_compile_flags) cmake_flags[k] = " ".join(v.split()) # set additional compiler flags - set_additional_flags('extra_compile_args', cmake_flags['KEYVI_CXX_FLAGS_ALL'].split(' ')) + set_additional_flags("extra_compile_args", cmake_flags["KEYVI_CXX_FLAGS_ALL"].split(" ")) # set defines - if cmake_flags['KEYVI_COMPILE_DEFINITIONS']: + if cmake_flags["KEYVI_COMPILE_DEFINITIONS"]: define_macros = [] - for macro in cmake_flags['KEYVI_COMPILE_DEFINITIONS'].split(' '): + for macro in cmake_flags["KEYVI_COMPILE_DEFINITIONS"].split(" "): if macro.count("=") == 0: define_macros.append((macro, None)) else: define_macros.append(macro.split("=", 1)) - set_additional_flags('define_macros', define_macros) + set_additional_flags("define_macros", define_macros) # set includes - if cmake_flags['KEYVI_INCLUDES']: - set_additional_flags('include_dirs', cmake_flags['KEYVI_INCLUDES'].split(' ')) + if cmake_flags["KEYVI_INCLUDES"]: + set_additional_flags("include_dirs", cmake_flags["KEYVI_INCLUDES"].split(" ")) # set link libraries - if cmake_flags['KEYVI_LINK_LIBRARIES_STATIC']: - if sys.platform == 'darwin': - set_additional_flags('libraries', cmake_flags['KEYVI_LINK_LIBRARIES_STATIC'].split(' ')) + if cmake_flags["KEYVI_LINK_LIBRARIES_STATIC"]: + if sys.platform == "darwin": + set_additional_flags("libraries", cmake_flags["KEYVI_LINK_LIBRARIES_STATIC"].split(" ")) else: - extra_link_arguments = ['-Wl,-Bstatic'] - for lib in cmake_flags['KEYVI_LINK_LIBRARIES_STATIC'].split(' '): + extra_link_arguments = ["-Wl,-Bstatic"] + for lib in cmake_flags["KEYVI_LINK_LIBRARIES_STATIC"].split(" "): extra_link_arguments.append("-l{}".format(lib)) # reset to dynamic - extra_link_arguments.append('-Wl,-Bdynamic') - set_additional_flags('extra_link_args', extra_link_arguments) + extra_link_arguments.append("-Wl,-Bdynamic") + set_additional_flags("extra_link_args", extra_link_arguments) - if cmake_flags['KEYVI_LINK_LIBRARIES_DYNAMIC']: - set_additional_flags('libraries', cmake_flags['KEYVI_LINK_LIBRARIES_DYNAMIC'].split(' ')) + if cmake_flags["KEYVI_LINK_LIBRARIES_DYNAMIC"]: + set_additional_flags("libraries", cmake_flags["KEYVI_LINK_LIBRARIES_DYNAMIC"].split(" ")) # set link args - if cmake_flags['KEYVI_LINK_FLAGS']: - set_additional_flags('extra_link_args', cmake_flags['KEYVI_LINK_FLAGS'].split(' ')) + if cmake_flags["KEYVI_LINK_FLAGS"]: + set_additional_flags("extra_link_args", cmake_flags["KEYVI_LINK_FLAGS"].split(" ")) return cmake_flags def set_additional_flags(key, additional_flags): - # patch the flags specified in key + # patch the flags specified in key for ext_m in ext_modules: flags = getattr(ext_m, key) + additional_flags setattr(ext_m, key, flags) @@ -195,10 +218,10 @@ def set_additional_flags(key, additional_flags): def patch_for_custom_zlib(zlib_root): for ext_m in ext_modules: - include_dirs = [path.join(zlib_root, "include")] + getattr(ext_m, 'include_dirs') - setattr(ext_m, 'include_dirs', include_dirs) - library_dirs = [path.join(zlib_root, "lib")] + getattr(ext_m, 'library_dirs') - setattr(ext_m, 'library_dirs', library_dirs) + include_dirs = [path.join(zlib_root, "include")] + getattr(ext_m, "include_dirs") + setattr(ext_m, "include_dirs", include_dirs) + library_dirs = [path.join(zlib_root, "lib")] + getattr(ext_m, "library_dirs") + setattr(ext_m, "library_dirs", library_dirs) with symlink_keyvi() as (pykeyvi_source_path, keyvi_source_path): @@ -207,38 +230,41 @@ def patch_for_custom_zlib(zlib_root): dictionary_sources = path.abspath(keyvi_cpp_link) - additional_compile_flags = '' + additional_compile_flags = "" # re-map the source files in the debug symbol tables to there original location so that stepping in a debugger works if pykeyvi_source_path is not None: - additional_compile_flags += ' -fdebug-prefix-map={}={}'.format(pykeyvi_source_path, keyvi_source_path) + additional_compile_flags += " -fdebug-prefix-map={}={}".format( + pykeyvi_source_path, keyvi_source_path + ) - link_library_dirs = [ - keyvi_build_dir, - '/usr/local/lib/', # as of 17/07/2022 Python 3.10 build on GH actions needs '/usr/local/lib/' link library dir - '/opt/homebrew/lib' - ] + link_library_dirs = [keyvi_build_dir] + + # if _CMAKE_PREFIX_PATH is set, append the lib subfolder for linking + cmake_prefix_path = os.environ.get("_CMAKE_PREFIX_PATH") + + if cmake_prefix_path is not None: + link_library_dirs.append(os.path.join(cmake_prefix_path, "lib")) + else: + # as of 17/07/2022 Python 3.10 build on GH actions needs '/usr/local/lib/' link library dir + link_library_dirs.append("/usr/local/lib/") + link_library_dirs.append("/opt/homebrew/lib") ######################### # Custom 'build' command ######################### - custom_user_options = [('mode=', - None, - "build mode."), - ('zlib-root=', - None, - "zlib installation root"), - ] + custom_user_options = [ + ("mode=", None, "build mode."), + ("zlib-root=", None, "zlib installation root"), + ] class custom_opts: - parent = None def initialize_options(self): self.parent.initialize_options(self) self.mode = None - self.staticlinkboost = False self.zlib_root = None self.options = {} @@ -249,10 +275,11 @@ def load_options(self): f = open(path.join(keyvi_build_dir, "custom_opts"), "r") self.options = json.loads(f.readline()) return - except: pass - self.options['mode'] = "release" if not self.mode else self.mode + except: + pass + self.options["mode"] = "release" if not self.mode else self.mode if self.zlib_root: - self.options['zlib_root'] = self.zlib_root + self.options["zlib_root"] = self.zlib_root def save_options(self): # store the options @@ -261,7 +288,23 @@ def save_options(self): def run(self): self.load_options() - self.cmake_flags = cmake_configure(keyvi_build_dir, self.options['mode'], self.options.get('zlib_root'), additional_compile_flags) + + # cross-compilation support for cibuildwheel for building M1 targets on x86_64 + osx_architectures = None + archflags = os.environ.get("ARCHFLAGS") + if archflags is not None: + osx_architectures = ";".join(set(archflags.split()) & {"x86_64", "arm64"}) + + cmake_prefix_path = os.environ.get("_CMAKE_PREFIX_PATH") + + self.cmake_flags = cmake_configure( + keyvi_build_dir, + self.options["mode"], + self.options.get("zlib_root"), + additional_compile_flags, + osx_architectures, + cmake_prefix_path, + ) self.save_options() self.parent.run(self) @@ -270,7 +313,6 @@ class build(custom_opts, _build.build): user_options = _build.build.user_options + custom_user_options class sdist(_sdist.sdist): - def run(self): clean_pykeyvi_build_directory() generate_pykeyvi_source() @@ -281,7 +323,6 @@ class bdist(custom_opts, _bdist.bdist): user_options = _bdist.bdist.user_options + custom_user_options class clean(_clean.clean): - def run(self): clean_pykeyvi_build_directory() _clean.clean.run(self) @@ -295,9 +336,11 @@ class bdist_wheel(custom_opts, _bdist_wheel.bdist_wheel): user_options = _bdist_wheel.bdist_wheel.user_options + custom_user_options have_wheel = True - except: None + except: + None class build_cxx(_build_ext.build_ext): + description = "customized for keyvi: " + _build_ext.build_ext.description def initialize_options(self): _build_ext.build_ext.initialize_options(self) @@ -306,38 +349,47 @@ def run(self): generate_pykeyvi_source() # custom zlib location - if 'zlib_root' in self.options: - patch_for_custom_zlib(self.options['zlib_root']) + if "zlib_root" in self.options: + patch_for_custom_zlib(self.options["zlib_root"]) - keyvi_build_cmd = 'cd {} && make -j {} bindings'.format(keyvi_build_dir, cpu_count) + keyvi_build_cmd = "cd {} && make -j {} bindings".format(keyvi_build_dir, cpu_count) - print ("Building keyvi C++ part: " + keyvi_build_cmd) + print("Building keyvi C++ part: " + keyvi_build_cmd) subprocess.call(keyvi_build_cmd, shell=True) _build_ext.build_ext.run(self) class build_ext(custom_opts, build_cxx): - parent = build_cxx user_options = build_cxx.user_options + custom_user_options - ext_modules = [Extension('keyvi._core', - include_dirs=[autowrap_data_dir], - language='c++', - sources=[pykeyvi_p_cpp], - library_dirs=link_library_dirs)] + ext_modules = [ + Extension( + "keyvi._core", + include_dirs=[autowrap_data_dir], + language="c++", + sources=[pykeyvi_p_cpp], + library_dirs=link_library_dirs, + ) + ] - PACKAGE_NAME = 'keyvi' - with open(os.path.join(here, 'description.md'), "rt", encoding="utf-8") as desc_f: + PACKAGE_NAME = "keyvi" + with open(os.path.join(here, "description.md"), "rt", encoding="utf-8") as desc_f: long_desc = desc_f.read() install_requires = [ - 'msgpack>=1.0.0', + "msgpack>=1.0.0", ] - commands = {'build_ext': build_ext, 'sdist': sdist, 'build': build, 'bdist': bdist, 'clean': clean} + commands = { + "build_ext": build_ext, + "sdist": sdist, + "build": build, + "bdist": bdist, + "clean": clean, + } if have_wheel: - commands['bdist_wheel'] = bdist_wheel + commands["bdist_wheel"] = bdist_wheel for e in ext_modules: e.cython_directives = {"embedsignature": True} @@ -345,41 +397,44 @@ class build_ext(custom_opts, build_cxx): setup( name=PACKAGE_NAME, version=VERSION, - description='Python package for keyvi', + description="Python package for keyvi", long_description=long_desc, long_description_content_type="text/markdown", - author='Hendrik Muhs', - author_email='hendrik.muhs@gmail.com', + author="Hendrik Muhs", + author_email="hendrik.muhs@gmail.com", license="ASL 2.0", cmdclass=commands, - scripts=['src/py/bin/keyvi'], - packages=['keyvi', - 'keyvi.cli', - 'keyvi.compiler', - 'keyvi.completion', - 'keyvi.dictionary', - 'keyvi.index', - 'keyvi.util', - 'keyvi.vector', - 'keyvi._pycore'], - package_dir={'': 'src/py'}, + scripts=["src/py/bin/keyvi"], + packages=[ + "keyvi", + "keyvi.cli", + "keyvi.compiler", + "keyvi.completion", + "keyvi.dictionary", + "keyvi.index", + "keyvi.util", + "keyvi.vector", + "keyvi._pycore", + ], + package_dir={"": "src/py"}, ext_modules=ext_modules, zip_safe=False, - url='http://keyvi.org', - download_url='https://github.com/KeyviDev/keyvi/tarball/v{}'.format(VERSION), - keywords=['FST'], + url="http://keyvi.org", + download_url="https://github.com/KeyviDev/keyvi/tarball/v{}".format(VERSION), + keywords=["FST"], classifiers=[ - 'Programming Language :: C++', - 'Programming Language :: Cython', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: Unix', + "Programming Language :: C++", + "Programming Language :: Cython", + "Programming Language :: Python", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Unix", ], install_requires=install_requires, )