From b43b8c2ba7623bb9d398b977500a0d44c00cbf6b Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 4 Dec 2022 19:25:39 +1100 Subject: [PATCH 01/25] GH-14828: treat patch files as binary (due to CRLF issues) --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitattributes b/.gitattributes index 1a5b156b491fd..e91661c424b4f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,5 @@ +*.patch binary + r/R/RcppExports.R linguist-generated=true r/R/arrowExports.R linguist-generated=true r/src/RcppExports.cpp linguist-generated=true From 0d31585cd2be0be4e0b19e99f6f0356f47efb7da Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 4 Dec 2022 19:26:24 +1100 Subject: [PATCH 02/25] GH-14828: sync with conda-forge recipe --- .../conda-recipes/arrow-cpp/bld-arrow.bat | 2 + .../conda-recipes/arrow-cpp/bld-pyarrow.bat | 4 + .../conda-recipes/arrow-cpp/build-arrow.sh | 1 + .../conda-recipes/arrow-cpp/build-pyarrow.sh | 10 + dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 307 ++++++++++++------ ...all-tests-requiring-gcs_server-fixtu.patch | 36 ++ 6 files changed, 254 insertions(+), 106 deletions(-) create mode 100644 dev/tasks/conda-recipes/arrow-cpp/patches/0001-definitely-skip-all-tests-requiring-gcs_server-fixtu.patch diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat index c8eacfb434853..b3c283ddb8f48 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat @@ -26,6 +26,7 @@ cmake -G "Ninja" ^ -DARROW_FILESYSTEM:BOOL=ON ^ -DARROW_FLIGHT:BOOL=ON ^ -DARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS:BOOL=ON ^ + -DARROW_FLIGHT_SQL:BOOL=ON ^ -DARROW_GANDIVA:BOOL=ON ^ -DARROW_GCS:BOOL=ON ^ -DARROW_HDFS:BOOL=ON ^ @@ -37,6 +38,7 @@ cmake -G "Ninja" ^ -DARROW_S3:BOOL=ON ^ -DARROW_SIMD_LEVEL:STRING=NONE ^ -DARROW_SUBSTRAIT:BOOL=ON ^ + -DARROW_USE_GLOG:BOOL=ON ^ -DARROW_WITH_BROTLI:BOOL=ON ^ -DARROW_WITH_BZ2:BOOL=ON ^ -DARROW_WITH_LZ4:BOOL=ON ^ diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat index bd20c79efe0b2..ed0e20554b766 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat @@ -25,6 +25,7 @@ SET PYARROW_WITH_HDFS=1 SET PYARROW_WITH_PARQUET=1 SET PYARROW_WITH_PARQUET_ENCRYPTION=1 SET PYARROW_WITH_S3=1 +SET PYARROW_WITH_SUBSTRAIT=1 SET PYARROW_CMAKE_GENERATOR=Ninja :: Enable CUDA support @@ -44,3 +45,6 @@ popd if [%PKG_NAME%] == [pyarrow] ( rd /s /q %SP_DIR%\pyarrow\tests ) + +:: generated by setup.py +rmdir .\python\build /s /q diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh index 3a26bd2ede1e4..5a0d26d6c7aaa 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -84,6 +84,7 @@ cmake -GNinja \ -DARROW_S3=ON \ -DARROW_SIMD_LEVEL=NONE \ -DARROW_SUBSTRAIT=ON \ + -DARROW_USE_GLOG=ON \ -DARROW_USE_LD_GOLD=ON \ -DARROW_WITH_BROTLI=ON \ -DARROW_WITH_BZ2=ON \ diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh index 76fdbce0c892d..692ffb7806d83 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh @@ -19,6 +19,7 @@ export PYARROW_WITH_PARQUET=1 export PYARROW_WITH_PARQUET_ENCRYPTION=1 export PYARROW_WITH_PLASMA=1 export PYARROW_WITH_S3=1 +export PYARROW_WITH_SUBSTRAIT=1 export PYARROW_CMAKE_GENERATOR=Ninja export PYARROW_CMAKE_OPTIONS="-DARROW_SIMD_LEVEL=NONE" BUILD_EXT_FLAGS="" @@ -30,6 +31,11 @@ else export PYARROW_WITH_CUDA=0 fi +# Resolve: Make Error at cmake_modules/SetupCxxFlags.cmake:338 (message): Unsupported arch flag: -march=. +if [[ "${target_platform}" == "linux-aarch64" ]]; then + export PYARROW_CMAKE_OPTIONS="-DARROW_ARMV8_ARCH=armv8-a ${PYARROW_CMAKE_OPTIONS}" +fi + if [[ "${target_platform}" == osx-* ]]; then # See https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk CXXFLAGS="${CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" @@ -50,3 +56,7 @@ $PYTHON setup.py \ if [[ "$PKG_NAME" == "pyarrow" ]]; then rm -r ${SP_DIR}/pyarrow/tests fi + +# generated by setup.py +rm -rf build +cd .. diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 9663e32cd26fd..0bb8359c3e235 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -7,32 +7,39 @@ {% set llvm_version = "14" %} package: - name: arrow-cpp-ext + name: apache-arrow version: {{ version }} source: path: ../../../../ + patches: + # switch off tests that require the as-yet unpackaged + # https://github.com/googleapis/storage-testbench + - patches/0001-definitely-skip-all-tests-requiring-gcs_server-fixtu.patch build: number: 0 # for cuda support, building with one version is enough to be compatible with # all later versions, since arrow is only using libcuda, and not libcudart. - skip: true # [(win or linux64) and cuda_compiler_version not in ("None", "10.2")] - skip: true # [linux and (aarch64 or ppc64le) and cuda_compiler_version not in ("None", "11.2")] - skip: true # [osx and cuda_compiler_version != "None"] - # CUDA builds on ppc64le currently run out of time on Travis CI. - # It may be possible to move these to cross-compilation, but this will take additional work. - # Hence this is skipped for now until this can be addressed. - skip: true # [linux and ppc64le and cuda_compiler_version != "None"] + skip: true # [cuda_compiler_version not in ("None", cuda_compiler_version_min)] + # temporary: skip CUDA on aarch/ppc until cross-compilation works, see + # https://github.com/conda-forge/conda-forge-ci-setup-feedstock/pull/210 + skip: true # [(aarch64 or ppc64le) and (cuda_compiler_version != "None")] run_exports: - - {{ pin_subpackage("arrow-cpp", max_pin="x.x.x") }} + - {{ pin_subpackage("libarrow", max_pin="x.x.x") }} outputs: - - name: arrow-cpp-proc + - name: apache-arrow-proc version: {{ build_ext_version }} build: number: {{ proc_build_number }} string: {{ build_ext }} + ignore_run_exports_from: + - openssl + requirements: + host: + # ensure we split the CI jobs per openssl version + - openssl test: commands: - exit 0 @@ -43,29 +50,46 @@ outputs: - LICENSE.txt summary: A meta-package to select Arrow build variant - - name: arrow-cpp + # compat output for old mutex-package naming + - name: arrow-cpp-proc + version: {{ build_ext_version }} + build: + number: {{ proc_build_number }} + string: {{ build_ext }} + ignore_run_exports_from: + - openssl + requirements: + host: + # ensure we split the CI jobs per openssl version + - openssl + run: + - apache-arrow-proc ={{ build_ext_version }}={{ build_ext }} + test: + commands: + - exit 0 + + - name: libarrow script: build-arrow.sh # [not win] script: bld-arrow.bat # [win] version: {{ version }} build: - string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} + string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} run_exports: - - {{ pin_subpackage("arrow-cpp", max_pin="x.x.x") }} + - {{ pin_subpackage("libarrow", max_pin="x.x.x") }} ignore_run_exports: - cudatoolkit track_features: {{ "[arrow-cuda]" if cuda_enabled else "" }} + missing_dso_whitelist: + - "*/libcuda.so.*" # [linux] + - "*/nvcuda.dll" # [win] requirements: build: - - python # [build_platform != target_platform] - - cross-python_{{ target_platform }} # [build_platform != target_platform] - - numpy # [build_platform != target_platform] - clangdev {{ llvm_version }} # [osx and arm64] - llvmdev {{ llvm_version }} # [osx and arm64] - gnuconfig # [osx and arm64] + - libgrpc - libprotobuf - - grpc-cpp - # aws-sdk-cpp 1.8.* doesn't work with a newer CMake - - cmake <3.22 + - cmake - autoconf # [unix] - ninja - make # [unix] @@ -73,11 +97,9 @@ outputs: - {{ compiler('cxx') }} - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] host: - # https://issues.apache.org/jira/browse/ARROW-15141 - - aws-sdk-cpp 1.8.186 - # abseil is only here to help conda pick the right constraints for pyarrow, see - # https://github.com/conda-forge/arrow-cpp-feedstock/pull/815#issuecomment-1216713245 - - libabseil + - clangdev {{ llvm_version }} + - llvmdev {{ llvm_version }} + - aws-sdk-cpp - boost-cpp >=1.70 - brotli - bzip2 @@ -85,17 +107,19 @@ outputs: - gflags - glog - google-cloud-cpp - - grpc-cpp + - libabseil + # since libgoogle-cloud is static on windows, see + # https://github.com/conda-forge/google-cloud-cpp-feedstock/pull/108, + # its dependencies leak into the build here + - libcrc32c # [win] + - libcurl # [win] + - libgrpc - libprotobuf - - clangdev {{ llvm_version }} - - llvmdev {{ llvm_version }} - libutf8proc - lz4-c - - numpy # gandiva depends on openssl - openssl - orc # [unix] - - python - rapidjson - re2 - snappy @@ -103,12 +127,48 @@ outputs: - xsimd - zlib - zstd - run: - - {{ pin_compatible('numpy', lower_bound='1.16') }} - - python run_constrained: - - arrow-cpp-proc * {{ build_ext }} - - cudatoolkit >=9.2 # [cuda_compiler_version != "None"] + - apache-arrow-proc =*={{ build_ext }} + - cudatoolkit >={{ cuda_compiler_version_min }} # [cuda_compiler_version != "None"] + # make sure we don't co-install with old version of old package name + - arrow-cpp ={{ version }} + + test: + commands: + {% set headers = [ + "arrow/api.h", "arrow/flight/types.h", "arrow/flight/sql/api.h", + "gandiva/engine.h", "parquet/api/reader.h" + ] %} + {% set headers = headers + ["plasma/client.h"] %} # [unix] + {% for each_header in headers %} + # headers + - test -f $PREFIX/include/{{ each_header }} || (echo "{{ each_header }} not found" && exit 1) # [unix] + - if not exist %LIBRARY_INC%\{{ "\\".join(each_header.split("/")) }} exit 1 # [win] + {% endfor %} + + {% set libs = (cuda_compiler_version != "None") * ["arrow_cuda"] + [ + "arrow", "arrow_dataset", "arrow_flight", "arrow_flight_sql", + "arrow_substrait", "gandiva", "parquet" + ] %} + {% set libs = libs + ["plasma"] %} # [unix] + {% for each_lib in libs %} + # shared + - test -f $PREFIX/lib/lib{{ each_lib }}.so # [linux] + - test -f $PREFIX/lib/lib{{ each_lib }}.dylib # [osx] + - if not exist %LIBRARY_BIN%\{{ each_lib }}.dll exit 1 # [win] + - if not exist %LIBRARY_LIB%\{{ each_lib }}.lib exit 1 # [win] + + # absence of static libraries + - test ! -f $PREFIX/lib/lib{{ each_lib }}.a # [unix] + - if exist %LIBRARY_LIB%\{{ each_lib }}_static.lib exit 1 # [win] + {% endfor %} + + # absence of arrow_cuda for CPU builds + - test ! -f $PREFIX/lib/libarrow_cuda.so # [(cuda_compiler_version == "None") and linux] + - test ! -f $PREFIX/lib/libarrow_cuda.a # [(cuda_compiler_version == "None") and linux] + - if exist %LIBRARY_BIN%\arrow_cuda.dll exit 1 # [(cuda_compiler_version == "None") and win] + - if exist %LIBRARY_LIB%\arrow_cuda.lib exit 1 # [(cuda_compiler_version == "None") and win] + - if exist %LIBRARY_LIB%\arrow_cuda_static.lib exit 1 # [(cuda_compiler_version == "None") and win] about: home: http://github.com/apache/arrow @@ -117,56 +177,23 @@ outputs: - LICENSE.txt summary: C++ libraries for Apache Arrow + # compat output for old naming scheme; switched for 10.0.0; keep for a few versions + - name: arrow-cpp + version: {{ version }} + build: + string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} + run_exports: + - {{ pin_subpackage("libarrow", max_pin="x.x.x") }} + requirements: + host: + # make sure the builds for different openssl versions (with the same PKG_BUILDNUM) have different hashes + - openssl + - {{ pin_subpackage('libarrow', exact=True) }} + run: + - {{ pin_subpackage('libarrow', exact=True) }} test: commands: - # headers - - test -f $PREFIX/include/arrow/api.h # [unix] - - test -f $PREFIX/include/arrow/flight/types.h # [unix] - - test -f $PREFIX/include/arrow/flight/sql/api.h # [unix] - - test -f $PREFIX/include/plasma/client.h # [unix] - - test -f $PREFIX/include/gandiva/engine.h # [unix] - - test -f $PREFIX/include/parquet/api/reader.h # [unix] - - if not exist %LIBRARY_INC%\\arrow\\api.h exit 1 # [win] - - if not exist %LIBRARY_INC%\\gandiva\\engine.h exit 1 # [win] - - if not exist %LIBRARY_INC%\\parquet\\api\\reader.h exit 1 # [win] - - # shared - - test -f $PREFIX/lib/libarrow.so # [linux] - - test -f $PREFIX/lib/libarrow_dataset.so # [linux] - - test -f $PREFIX/lib/libarrow_flight.so # [linux] - - test -f $PREFIX/lib/libarrow_flight_sql.so # [linux] - - test -f $PREFIX/lib/libparquet.so # [linux] - - test -f $PREFIX/lib/libgandiva.so # [linux] - - test -f $PREFIX/lib/libplasma.so # [linux] - - test -f $PREFIX/lib/libarrow_cuda${SHLIB_EXT} # [(cuda_compiler_version != "None") and unix] - - test ! -f $PREFIX/lib/libarrow_cuda${SHLIB_EXT} # [(cuda_compiler_version == "None") and unix] - - if not exist %PREFIX%\\Library\\bin\\arrow_cuda.dll exit 1 # [(cuda_compiler_version != "None") and win] - - if exist %PREFIX%\\Library\\bin\\arrow_cuda.dll exit 1 # [(cuda_compiler_version == "None") and win] - - test -f $PREFIX/lib/libarrow.dylib # [osx] - - test -f $PREFIX/lib/libarrow_dataset.dylib # [osx] - - test -f $PREFIX/lib/libgandiva.dylib # [osx] - - test -f $PREFIX/lib/libparquet.dylib # [osx] - - test -f $PREFIX/lib/libplasma.dylib # [osx] - - if not exist %PREFIX%\\Library\\bin\\arrow.dll exit 1 # [win] - - if not exist %PREFIX%\\Library\\bin\\arrow_dataset.dll exit 1 # [win] - - if not exist %PREFIX%\\Library\\bin\\arrow_flight.dll exit 1 # [win] - - if not exist %PREFIX%\\Library\\bin\\parquet.dll exit 1 # [win] - - if not exist %PREFIX%\\Library\\bin\\gandiva.dll exit 1 # [win] - - # absence of static libraries - - test ! -f $PREFIX/lib/libarrow.a # [unix] - - test ! -f $PREFIX/lib/libarrow_dataset.a # [unix] - - test ! -f $PREFIX/lib/libarrow_flight.a # [unix] - - test ! -f $PREFIX/lib/libarrow_flight_sql.a # [unix] - - test ! -f $PREFIX/lib/libplasma.a # [unix] - - test ! -f $PREFIX/lib/libparquet.a # [unix] - - test ! -f $PREFIX/lib/libgandiva.a # [unix] - - if exist %PREFIX%\\Library\\lib\\arrow_static.lib exit 1 # [win] - - if exist %PREFIX%\\Library\\lib\\arrow_dataset_static.lib exit 1 # [win] - - if exist %PREFIX%\\Library\\lib\\arrow_flight_static.lib exit 1 # [win] - - if exist %PREFIX%\\Library\\lib\\arrow_flight_sql_static.lib exit 1 # [win] - - if exist %PREFIX%\\Library\\lib\\parquet_static.lib exit 1 # [win] - - if exist %PREFIX%\\Library\\lib\\gandiva_static.lib exit 1 # [win] + - exit 0 - name: pyarrow script: build-pyarrow.sh # [not win] @@ -179,6 +206,14 @@ outputs: ignore_run_exports_from: - openssl track_features: {{ "[arrow-cuda]" if cuda_enabled else "" }} + missing_dso_whitelist: + # not actually missing, but installed into SP_DIR, see tests + - "*/libarrow_python.so.*" # [linux] + - "*/libarrow_python_flight.so.*" # [linux] + - "*/libarrow_python.*.dylib" # [osx] + - "*/libarrow_python_flight.*.dylib" # [osx] + - "*/arrow_python.dll" # [win] + - "*/arrow_python_flight.dll" # [win] requirements: build: - python # [build_platform != target_platform] @@ -190,35 +225,28 @@ outputs: - make # [unix] - {{ compiler('c') }} - {{ compiler('cxx') }} - # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda + # pyarrow does not require nvcc but it needs to link against libraries in libarrow=*=*cuda - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] host: - - {{ pin_subpackage('arrow-cpp', exact=True) }} + - {{ pin_subpackage('libarrow', exact=True) }} - clangdev {{ llvm_version }} - llvmdev {{ llvm_version }} - cython + - gflags # [unix] - numpy - openssl - python - setuptools - setuptools_scm - - six run: - - {{ pin_subpackage('arrow-cpp', exact=True) }} - - {{ pin_compatible('numpy', lower_bound='1.16') }} + - {{ pin_subpackage('libarrow', exact=True) }} + - {{ pin_compatible('numpy') }} # empty parquet-cpp metapackage, force old versions to be uninstalled - parquet-cpp 1.5.1.* - python run_constrained: - - arrow-cpp-proc * {{ build_ext }} - - cudatoolkit >=9.2 # [cuda_compiler_version != "None"] - - about: - home: http://github.com/apache/arrow - license: Apache-2.0 - license_file: - - LICENSE.txt - summary: Python libraries for Apache Arrow + - apache-arrow-proc =*={{ build_ext }} + - cudatoolkit >={{ cuda_compiler_version_min }} # [cuda_compiler_version != "None"] test: files: @@ -241,12 +269,30 @@ outputs: # However, we check below for (at least) the presence of a correctly-compiled module - pyarrow.cuda # [cuda_compiler_version != "None" and not win] commands: + # libraries that depend on python (and hence aren't in libarrow itself) + - test -f ${SP_DIR}/pyarrow/libarrow_python.so # [linux] + - test -f ${SP_DIR}/pyarrow/libarrow_python_flight.so # [linux] + - test -f ${SP_DIR}/pyarrow/libarrow_python.dylib # [osx] + - test -f ${SP_DIR}/pyarrow/libarrow_python_flight.dylib # [osx] + - if not exist %SP_DIR%\pyarrow\arrow_python.dll exit 1 # [win] + - if not exist %SP_DIR%\pyarrow\arrow_python_flight.dll exit 1 # [win] + + - test -f ${SP_DIR}/pyarrow/include/arrow/python/pyarrow.h # [unix] + - if not exist %SP_DIR%\pyarrow\include\arrow\python\pyarrow.h exit 1 # [win] + - test ! -f ${SP_DIR}/pyarrow/tests/test_array.py # [unix] - if exist %SP_DIR%/pyarrow/tests/test_array.py exit 1 # [win] # Need to remove dot from PY_VER; %MYVAR:x=y% replaces "x" in %MYVAR% with "y" - if not exist %SP_DIR%/pyarrow/_cuda.cp%PY_VER:.=%-win_amd64.pyd exit 1 # [win and cuda_compiler_version != "None"] - python test_read_parquet.py + about: + home: http://github.com/apache/arrow + license: Apache-2.0 + license_file: + - LICENSE.txt + summary: Python libraries for Apache Arrow + - name: pyarrow-tests script: build-pyarrow.sh # [not win] script: bld-pyarrow.bat # [win] @@ -269,10 +315,10 @@ outputs: - make # [unix] - {{ compiler('c') }} - {{ compiler('cxx') }} - # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda + # pyarrow does not require nvcc but it needs to link against libraries in libarrow=*=*cuda - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] host: - - {{ pin_subpackage('arrow-cpp', exact=True) }} + - {{ pin_subpackage('libarrow', exact=True) }} - {{ pin_subpackage('pyarrow', exact=True) }} - clangdev {{ llvm_version }} - llvmdev {{ llvm_version }} @@ -282,13 +328,66 @@ outputs: - python - setuptools - setuptools_scm - - six run: - {{ pin_subpackage('pyarrow', exact=True) }} - python run_constrained: - - arrow-cpp-proc * {{ build_ext }} - - cudatoolkit >=9.2 # [cuda_compiler_version != "None"] + - apache-arrow-proc =*={{ build_ext }} + - cudatoolkit >={{ cuda_compiler_version_min }} # [cuda_compiler_version != "None"] + + test: + requires: + - pytest + - pytest-lazy-fixture + - backports.zoneinfo # [py<39] + - cffi + - cloudpickle + - cython + - fastparquet + - fsspec + - hypothesis + - pandas + # not all arches available in conda-forge; osx-arm not tested anyway + - pytorch * # [unix and not ppc64le and py<311] + - scipy + # we're not building java bindings + # - jpype1 + # doesn't get picked up correctly + # - libhdfs3 + # causes segfaults + # - sparse + # not rebuilt for grpc 1.49 yet + # - tensorflow + source_files: + - testing/data + commands: + - cd ${SP_DIR}/pyarrow/tests # [unix] + - cd %SP_DIR%\pyarrow\tests # [win] + - export ARROW_TEST_DATA="${SRC_DIR}/testing/data" # [unix] + - set "ARROW_TEST_DATA=%SRC_DIR%\testing\data" # [win] + + {% set tests_to_skip = "_not_a_real_test" %} + # we do not have GPUs in CI --> cannot test cuda + {% set tests_to_skip = tests_to_skip + " or test_cuda" %} + # skip tests that raise SIGINT and crash the test suite + {% set tests_to_skip = tests_to_skip + " or (test_csv and test_cancellation)" %} # [linux] + {% set tests_to_skip = tests_to_skip + " or (test_flight and test_interrupt)" %} # [linux] + # skip tests that make invalid(-for-conda) assumptions about the compilers setup + {% set tests_to_skip = tests_to_skip + " or test_cython_api" %} # [unix] + {% set tests_to_skip = tests_to_skip + " or test_visit_strings" %} # [unix] + # skip tests that cannot succeed in emulation + {% set tests_to_skip = tests_to_skip + " or test_debug_memory_pool_disabled" %} # [aarch64 or ppc64le] + {% set tests_to_skip = tests_to_skip + " or test_env_var_io_thread_count" %} # [aarch64 or ppc64le] + # vvvvvvv TESTS THAT SHOULDN'T HAVE TO BE SKIPPED vvvvvvv + # failing on linux with "OSError: Could not connect to socket /tmp/[...]/plasma.sock" + {% set tests_to_skip = tests_to_skip + " or test_plasma" %} # [linux] + # "Unsupported backend 'nonexistent' specified in ARROW_DEFAULT_MEMORY_POOL" + {% set tests_to_skip = tests_to_skip + " or (test_memory and test_env_var)" %} # [unix] + # test is broken; header is in $PREFIX, not $SP_DIR + {% set tests_to_skip = tests_to_skip + " or (test_misc and test_get_include)" %} # [unix] + - pytest -v -rfEs -k "not ({{ tests_to_skip }})" # [not ppc64le] + # NOTE: test suite is skipped on ppc due to bugs in QEMU code that cause + # CI to fail, even though the tests would run through on native hardware about: home: http://github.com/apache/arrow @@ -297,11 +396,6 @@ outputs: - LICENSE.txt summary: Python test files for Apache Arrow - test: - commands: - - test -f ${SP_DIR}/pyarrow/tests/test_array.py # [unix] - - if not exist %SP_DIR%/pyarrow/tests/test_array.py exit 1 # [win] - about: home: http://github.com/apache/arrow license: Apache-2.0 @@ -326,3 +420,4 @@ extra: - nealrichardson - jakirkham - h-vetinari + feedstock-name: arrow-cpp diff --git a/dev/tasks/conda-recipes/arrow-cpp/patches/0001-definitely-skip-all-tests-requiring-gcs_server-fixtu.patch b/dev/tasks/conda-recipes/arrow-cpp/patches/0001-definitely-skip-all-tests-requiring-gcs_server-fixtu.patch new file mode 100644 index 0000000000000..7dd04b616d053 --- /dev/null +++ b/dev/tasks/conda-recipes/arrow-cpp/patches/0001-definitely-skip-all-tests-requiring-gcs_server-fixtu.patch @@ -0,0 +1,36 @@ +From 2549974357216cab621d214582484e9b056c2984 Mon Sep 17 00:00:00 2001 +From: "H. Vetinari" +Date: Wed, 23 Nov 2022 17:10:52 +1100 +Subject: [PATCH] definitely skip all tests requiring gcs_server fixture + +since we don't have https://github.com/googleapis/storage-testbench +in conda-forge so far, which is responsible for the `python -m testbench` +--- + python/pyarrow/tests/conftest.py | 14 +------------- + 1 file changed, 1 insertion(+), 13 deletions(-) + +diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py +index 154ef79dc..d1d688814 100644 +--- a/python/pyarrow/tests/conftest.py ++++ b/python/pyarrow/tests/conftest.py +@@ -183,16 +183,4 @@ def gcs_server(): + env = os.environ.copy() + args = [sys.executable, '-m', 'testbench', '--port', str(port)] + proc = None +- try: +- proc = subprocess.Popen(args, env=env) +- except OSError as e: +- pytest.skip(f"Command {args} failed to execute: {e}") +- else: +- yield { +- 'connection': ('localhost', port), +- 'process': proc, +- } +- finally: +- if proc is not None: +- proc.kill() +- proc.wait() ++ pytest.skip(f"Tests depending on gcs_server disabled for conda builds for now") +-- +2.38.1.windows.1 + From a123df8a9cf0340b72018d90c5509d4ba96a5df3 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 4 Dec 2022 19:50:02 +1100 Subject: [PATCH 03/25] GH-14828: sync dev/tasks/conda-recipes/.ci_support with conda-forge feedstock --- ...rsion10numpy1.20python3.7.____cpython.yaml | 74 ------ ...rsion10numpy1.20python3.9.____cpython.yaml | 74 ------ ...ersion7numpy1.20python3.7.____cpython.yaml | 74 ------ ...ersion7numpy1.20python3.9.____cpython.yaml | 74 ------ ...uda_compiler_version10.2openssl1.1.1.yaml} | 22 +- ...64_cuda_compiler_version10.2openssl3.yaml} | 24 +- ...uda_compiler_versionNoneopenssl1.1.1.yaml} | 22 +- ...64_cuda_compiler_versionNoneopenssl3.yaml} | 24 +- ...ionNonenumpy1.20python3.7.____cpython.yaml | 78 ------ ...ionNonenumpy1.20python3.9.____cpython.yaml | 78 ------ ...uda_compiler_versionNoneopenssl1.1.1.yaml} | 22 +- ...64_cuda_compiler_versionNoneopenssl3.yaml} | 24 +- ...uda_compiler_versionNoneopenssl1.1.1.yaml} | 20 +- ...le_cuda_compiler_versionNoneopenssl3.yaml} | 22 +- ...pc64le_numpy1.20python3.7.____cpython.yaml | 72 ------ ...pc64le_numpy1.20python3.9.____cpython.yaml | 72 ------ ...osx_64_numpy1.20python3.7.____cpython.yaml | 69 ------ ...osx_64_numpy1.20python3.9.____cpython.yaml | 69 ------ ..._cpython.yaml => osx_64_openssl1.1.1.yaml} | 20 +- ....____cpython.yaml => osx_64_openssl3.yaml} | 22 +- ..._arm64_numpy1.20python3.8.____cpython.yaml | 69 ------ ...ython.yaml => osx_arm64_openssl1.1.1.yaml} | 20 +- ...__cpython.yaml => osx_arm64_openssl3.yaml} | 22 +- ...ion10.2numpy1.20python3.7.____cpython.yaml | 59 ----- ...uda_compiler_version10.2openssl1.1.1.yaml} | 24 +- ...64_cuda_compiler_version10.2openssl3.yaml} | 28 ++- ...ionNonenumpy1.20python3.7.____cpython.yaml | 59 ----- ...ionNonenumpy1.20python3.8.____cpython.yaml | 59 ----- ...ionNonenumpy1.20python3.9.____cpython.yaml | 59 ----- ...uda_compiler_versionNoneopenssl1.1.1.yaml} | 24 +- ...64_cuda_compiler_versionNoneopenssl3.yaml} | 26 +- dev/tasks/tasks.yml | 225 +++++++++++------- 32 files changed, 365 insertions(+), 1265 deletions(-) delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.7.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.9.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.20python3.7.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.20python3.9.____cpython.yaml rename dev/tasks/conda-recipes/.ci_support/{linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.21python3.10.____cpython.yaml => linux_64_cuda_compiler_version10.2openssl1.1.1.yaml} (82%) rename dev/tasks/conda-recipes/.ci_support/{linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.20python3.8.____cpython.yaml => linux_64_cuda_compiler_version10.2openssl3.yaml} (81%) rename dev/tasks/conda-recipes/.ci_support/{linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.21python3.10.____cpython.yaml => linux_64_cuda_compiler_versionNoneopenssl1.1.1.yaml} (82%) rename dev/tasks/conda-recipes/.ci_support/{linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.8.____cpython.yaml => linux_64_cuda_compiler_versionNoneopenssl3.yaml} (81%) delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNonenumpy1.20python3.7.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNonenumpy1.20python3.9.____cpython.yaml rename dev/tasks/conda-recipes/.ci_support/{linux_aarch64_cuda_compiler_versionNonenumpy1.21python3.10.____cpython.yaml => linux_aarch64_cuda_compiler_versionNoneopenssl1.1.1.yaml} (85%) rename dev/tasks/conda-recipes/.ci_support/{linux_aarch64_cuda_compiler_versionNonenumpy1.20python3.8.____cpython.yaml => linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml} (85%) rename dev/tasks/conda-recipes/.ci_support/{linux_ppc64le_numpy1.21python3.10.____cpython.yaml => linux_ppc64le_cuda_compiler_versionNoneopenssl1.1.1.yaml} (84%) rename dev/tasks/conda-recipes/.ci_support/{linux_ppc64le_numpy1.20python3.8.____cpython.yaml => linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml} (84%) delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_ppc64le_numpy1.20python3.7.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_ppc64le_numpy1.20python3.9.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.20python3.7.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.20python3.9.____cpython.yaml rename dev/tasks/conda-recipes/.ci_support/{osx_64_numpy1.21python3.10.____cpython.yaml => osx_64_openssl1.1.1.yaml} (83%) rename dev/tasks/conda-recipes/.ci_support/{osx_64_numpy1.20python3.8.____cpython.yaml => osx_64_openssl3.yaml} (83%) delete mode 100644 dev/tasks/conda-recipes/.ci_support/osx_arm64_numpy1.20python3.8.____cpython.yaml rename dev/tasks/conda-recipes/.ci_support/{osx_arm64_numpy1.21python3.10.____cpython.yaml => osx_arm64_openssl1.1.1.yaml} (83%) rename dev/tasks/conda-recipes/.ci_support/{osx_arm64_numpy1.20python3.9.____cpython.yaml => osx_arm64_openssl3.yaml} (83%) delete mode 100644 dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.20python3.7.____cpython.yaml rename dev/tasks/conda-recipes/.ci_support/{win_64_cuda_compiler_version10.2numpy1.20python3.8.____cpython.yaml => win_64_cuda_compiler_version10.2openssl1.1.1.yaml} (73%) rename dev/tasks/conda-recipes/.ci_support/{win_64_cuda_compiler_versionNonenumpy1.21python3.10.____cpython.yaml => win_64_cuda_compiler_version10.2openssl3.yaml} (71%) delete mode 100644 dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.20python3.7.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.20python3.8.____cpython.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.20python3.9.____cpython.yaml rename dev/tasks/conda-recipes/.ci_support/{win_64_cuda_compiler_version10.2numpy1.21python3.10.____cpython.yaml => win_64_cuda_compiler_versionNoneopenssl1.1.1.yaml} (73%) rename dev/tasks/conda-recipes/.ci_support/{win_64_cuda_compiler_version10.2numpy1.20python3.9.____cpython.yaml => win_64_cuda_compiler_versionNoneopenssl3.yaml} (72%) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.7.____cpython.yaml deleted file mode 100644 index 103484667f830..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.7.____cpython.yaml +++ /dev/null @@ -1,74 +0,0 @@ -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '10' -cdt_name: -- cos6 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '10' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' -libabseil: -- '20220623.0' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.7.6 -pin_run_as_build: - bzip2: - max_pin: x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.7.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- linux-64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version - - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.9.____cpython.yaml deleted file mode 100644 index 3652c6ed97318..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.9.____cpython.yaml +++ /dev/null @@ -1,74 +0,0 @@ -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '10' -cdt_name: -- cos6 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '10' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' -libabseil: -- '20220623.0' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.7.6 -pin_run_as_build: - bzip2: - max_pin: x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.9.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- linux-64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version - - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.20python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.20python3.7.____cpython.yaml deleted file mode 100644 index 6a3f12704850e..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.20python3.7.____cpython.yaml +++ /dev/null @@ -1,74 +0,0 @@ -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '7' -cdt_name: -- cos6 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.2' -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-cuda:10.2 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' -libabseil: -- '20220623.0' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.7.6 -pin_run_as_build: - bzip2: - max_pin: x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.7.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- linux-64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version - - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.20python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.20python3.9.____cpython.yaml deleted file mode 100644 index 53307c2603946..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.20python3.9.____cpython.yaml +++ /dev/null @@ -1,74 +0,0 @@ -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '7' -cdt_name: -- cos6 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.2' -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-cuda:10.2 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' -libabseil: -- '20220623.0' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.7.6 -pin_run_as_build: - bzip2: - max_pin: x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.9.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- linux-64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version - - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.21python3.10.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl1.1.1.yaml similarity index 82% rename from dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.21python3.10.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl1.1.1.yaml index 6e1f0354f1ec8..09e8dfb7372b3 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.21python3.10.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl1.1.1.yaml @@ -1,3 +1,5 @@ +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -16,6 +18,8 @@ cuda_compiler: - nvcc cuda_compiler_version: - '10.2' +cuda_compiler_version_min: +- '10.2' cxx_compiler: - gxx cxx_compiler_version: @@ -27,31 +31,33 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: - 1.9.3 numpy: - '1.21' +- '1.23' +- '1.20' +- '1.20' openssl: - 1.1.1 orc: -- 1.7.6 +- 1.8.0 pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: - 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython re2: - 2022.06.01 snappy: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.20python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml similarity index 81% rename from dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.20python3.8.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml index 6ee37c66d15fd..99022e84887e8 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.20python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml @@ -1,3 +1,5 @@ +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -16,6 +18,8 @@ cuda_compiler: - nvcc cuda_compiler_version: - '10.2' +cuda_compiler_version_min: +- '10.2' cxx_compiler: - gxx cxx_compiler_version: @@ -27,31 +31,33 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: - 1.9.3 numpy: +- '1.21' +- '1.23' +- '1.20' - '1.20' openssl: -- 1.1.1 +- '3' orc: -- 1.7.6 +- 1.8.0 pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: +- 3.10.* *_cpython +- 3.11.* *_cpython - 3.8.* *_cpython +- 3.9.* *_cpython re2: - 2022.06.01 snappy: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.21python3.10.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl1.1.1.yaml similarity index 82% rename from dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.21python3.10.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl1.1.1.yaml index ea90f70614af9..edba301f4a1f0 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.21python3.10.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl1.1.1.yaml @@ -1,3 +1,5 @@ +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -16,6 +18,8 @@ cuda_compiler: - nvcc cuda_compiler_version: - None +cuda_compiler_version_min: +- '10.2' cxx_compiler: - gxx cxx_compiler_version: @@ -27,31 +31,33 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: - 1.9.3 numpy: - '1.21' +- '1.23' +- '1.20' +- '1.20' openssl: - 1.1.1 orc: -- 1.7.6 +- 1.8.0 pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: - 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython re2: - 2022.06.01 snappy: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml similarity index 81% rename from dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.8.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml index 892afa7ea6420..120d8eb622b57 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml @@ -1,3 +1,5 @@ +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -16,6 +18,8 @@ cuda_compiler: - nvcc cuda_compiler_version: - None +cuda_compiler_version_min: +- '10.2' cxx_compiler: - gxx cxx_compiler_version: @@ -27,31 +31,33 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: - 1.9.3 numpy: +- '1.21' +- '1.23' +- '1.20' - '1.20' openssl: -- 1.1.1 +- '3' orc: -- 1.7.6 +- 1.8.0 pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: +- 3.10.* *_cpython +- 3.11.* *_cpython - 3.8.* *_cpython +- 3.9.* *_cpython re2: - 2022.06.01 snappy: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNonenumpy1.20python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNonenumpy1.20python3.7.____cpython.yaml deleted file mode 100644 index fcd7d6c04d53a..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNonenumpy1.20python3.7.____cpython.yaml +++ /dev/null @@ -1,78 +0,0 @@ -BUILD: -- aarch64-conda_cos7-linux-gnu -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '10' -cdt_arch: -- aarch64 -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '10' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' -libabseil: -- '20220623.0' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.7.6 -pin_run_as_build: - bzip2: - max_pin: x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.7.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- linux-aarch64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version - - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNonenumpy1.20python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNonenumpy1.20python3.9.____cpython.yaml deleted file mode 100644 index 2532c0fd7c42c..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNonenumpy1.20python3.9.____cpython.yaml +++ /dev/null @@ -1,78 +0,0 @@ -BUILD: -- aarch64-conda_cos7-linux-gnu -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '10' -cdt_arch: -- aarch64 -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '10' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' -libabseil: -- '20220623.0' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.7.6 -pin_run_as_build: - bzip2: - max_pin: x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.9.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- linux-aarch64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version - - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNonenumpy1.21python3.10.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl1.1.1.yaml similarity index 85% rename from dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNonenumpy1.21python3.10.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl1.1.1.yaml index a2b090f0bcfb0..7ec61b35a0f59 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNonenumpy1.21python3.10.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl1.1.1.yaml @@ -1,5 +1,7 @@ BUILD: - aarch64-conda_cos7-linux-gnu +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -16,8 +18,6 @@ channel_sources: - conda-forge channel_targets: - conda-forge main -cuda_compiler: -- nvcc cuda_compiler_version: - None cxx_compiler: @@ -31,31 +31,33 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: - 1.9.3 numpy: - '1.21' +- '1.23' +- '1.20' +- '1.20' openssl: - 1.1.1 orc: -- 1.7.6 +- 1.8.0 pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: - 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython re2: - 2022.06.01 snappy: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNonenumpy1.20python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml similarity index 85% rename from dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNonenumpy1.20python3.8.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml index 32d570e52a104..fa91ba903dec6 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNonenumpy1.20python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml @@ -1,5 +1,7 @@ BUILD: - aarch64-conda_cos7-linux-gnu +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -16,8 +18,6 @@ channel_sources: - conda-forge channel_targets: - conda-forge main -cuda_compiler: -- nvcc cuda_compiler_version: - None cxx_compiler: @@ -31,31 +31,33 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: - 1.9.3 numpy: +- '1.21' +- '1.23' +- '1.20' - '1.20' openssl: -- 1.1.1 +- '3' orc: -- 1.7.6 +- 1.8.0 pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: +- 3.10.* *_cpython +- 3.11.* *_cpython - 3.8.* *_cpython +- 3.9.* *_cpython re2: - 2022.06.01 snappy: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_numpy1.21python3.10.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl1.1.1.yaml similarity index 84% rename from dev/tasks/conda-recipes/.ci_support/linux_ppc64le_numpy1.21python3.10.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl1.1.1.yaml index a1fd913c033a6..c5a546f91195e 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_numpy1.21python3.10.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl1.1.1.yaml @@ -1,3 +1,5 @@ +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -25,31 +27,33 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: - 1.9.3 numpy: - '1.21' +- '1.23' +- '1.20' +- '1.20' openssl: - 1.1.1 orc: -- 1.7.6 +- 1.8.0 pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: - 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython re2: - 2022.06.01 snappy: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_numpy1.20python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml similarity index 84% rename from dev/tasks/conda-recipes/.ci_support/linux_ppc64le_numpy1.20python3.8.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml index df80774cd82e0..16c5692757d17 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_numpy1.20python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml @@ -1,3 +1,5 @@ +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -25,31 +27,33 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: - 1.9.3 numpy: +- '1.21' +- '1.23' +- '1.20' - '1.20' openssl: -- 1.1.1 +- '3' orc: -- 1.7.6 +- 1.8.0 pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: +- 3.10.* *_cpython +- 3.11.* *_cpython - 3.8.* *_cpython +- 3.9.* *_cpython re2: - 2022.06.01 snappy: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_numpy1.20python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_numpy1.20python3.7.____cpython.yaml deleted file mode 100644 index 9ece142ae8b8c..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_numpy1.20python3.7.____cpython.yaml +++ /dev/null @@ -1,72 +0,0 @@ -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '10' -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '10' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' -libabseil: -- '20220623.0' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.7.6 -pin_run_as_build: - bzip2: - max_pin: x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.7.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- linux-ppc64le -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version - - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_numpy1.20python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_numpy1.20python3.9.____cpython.yaml deleted file mode 100644 index f697be7cb350c..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_numpy1.20python3.9.____cpython.yaml +++ /dev/null @@ -1,72 +0,0 @@ -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '10' -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '10' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' -libabseil: -- '20220623.0' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.7.6 -pin_run_as_build: - bzip2: - max_pin: x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.9.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- linux-ppc64le -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version - - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.20python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.20python3.7.____cpython.yaml deleted file mode 100644 index f9aa6765bb3b9..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.20python3.7.____cpython.yaml +++ /dev/null @@ -1,69 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '10.9' -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- clang -c_compiler_version: -- '14' -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- clangxx -cxx_compiler_version: -- '14' -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' -libabseil: -- '20220623.0' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -macos_machine: -- x86_64-apple-darwin13.4.0 -numpy: -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.7.6 -pin_run_as_build: - bzip2: - max_pin: x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.7.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- osx-64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.20python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.20python3.9.____cpython.yaml deleted file mode 100644 index 317fba95fdaba..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.20python3.9.____cpython.yaml +++ /dev/null @@ -1,69 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '10.9' -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- clang -c_compiler_version: -- '14' -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- clangxx -cxx_compiler_version: -- '14' -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' -libabseil: -- '20220623.0' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -macos_machine: -- x86_64-apple-darwin13.4.0 -numpy: -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.7.6 -pin_run_as_build: - bzip2: - max_pin: x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.9.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- osx-64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.21python3.10.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_openssl1.1.1.yaml similarity index 83% rename from dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.21python3.10.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/osx_64_openssl1.1.1.yaml index 3d6052d0f9354..a52bf9bbd3558 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.21python3.10.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_openssl1.1.1.yaml @@ -1,5 +1,7 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -23,11 +25,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: @@ -36,20 +38,22 @@ macos_machine: - x86_64-apple-darwin13.4.0 numpy: - '1.21' +- '1.23' +- '1.20' +- '1.20' openssl: - 1.1.1 orc: -- 1.7.6 +- 1.8.0 pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: - 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython re2: - 2022.06.01 snappy: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.20python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml similarity index 83% rename from dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.20python3.8.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml index b2461b7e6cb14..f47c333a5b2b1 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.20python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml @@ -1,5 +1,7 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -23,11 +25,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: @@ -35,21 +37,23 @@ lz4_c: macos_machine: - x86_64-apple-darwin13.4.0 numpy: +- '1.21' +- '1.23' +- '1.20' - '1.20' openssl: -- 1.1.1 +- '3' orc: -- 1.7.6 +- 1.8.0 pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: +- 3.10.* *_cpython +- 3.11.* *_cpython - 3.8.* *_cpython +- 3.9.* *_cpython re2: - 2022.06.01 snappy: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_numpy1.20python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_numpy1.20python3.8.____cpython.yaml deleted file mode 100644 index 7dd2d05dbdb45..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_numpy1.20python3.8.____cpython.yaml +++ /dev/null @@ -1,69 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '11.0' -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- clang -c_compiler_version: -- '14' -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- clangxx -cxx_compiler_version: -- '14' -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' -libabseil: -- '20220623.0' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -macos_machine: -- arm64-apple-darwin20.0.0 -numpy: -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.7.6 -pin_run_as_build: - bzip2: - max_pin: x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.8.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- osx-arm64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_numpy1.21python3.10.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl1.1.1.yaml similarity index 83% rename from dev/tasks/conda-recipes/.ci_support/osx_arm64_numpy1.21python3.10.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl1.1.1.yaml index 0e4ea0d725c0c..59128de87d6c4 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_numpy1.21python3.10.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl1.1.1.yaml @@ -1,5 +1,7 @@ MACOSX_DEPLOYMENT_TARGET: - '11.0' +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -23,11 +25,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: @@ -36,20 +38,22 @@ macos_machine: - arm64-apple-darwin20.0.0 numpy: - '1.21' +- '1.23' +- '1.20' +- '1.20' openssl: - 1.1.1 orc: -- 1.7.6 +- 1.8.0 pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: - 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython re2: - 2022.06.01 snappy: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_numpy1.20python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml similarity index 83% rename from dev/tasks/conda-recipes/.ci_support/osx_arm64_numpy1.20python3.9.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml index 4adda2707a780..4c08c312f839c 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_numpy1.20python3.9.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml @@ -1,5 +1,7 @@ MACOSX_DEPLOYMENT_TARGET: - '11.0' +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -23,11 +25,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: @@ -35,20 +37,22 @@ lz4_c: macos_machine: - arm64-apple-darwin20.0.0 numpy: +- '1.21' +- '1.23' +- '1.20' - '1.20' openssl: -- 1.1.1 +- '3' orc: -- 1.7.6 +- 1.8.0 pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: +- 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython - 3.9.* *_cpython re2: - 2022.06.01 diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.20python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.20python3.7.____cpython.yaml deleted file mode 100644 index b3a8a8c278e9e..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.20python3.7.____cpython.yaml +++ /dev/null @@ -1,59 +0,0 @@ -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- vs2019 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.2' -cxx_compiler: -- vs2019 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' -libabseil: -- '20220623.0' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.20' -openssl: -- 1.1.1 -pin_run_as_build: - bzip2: - max_pin: x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.7.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- win-64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.20python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl1.1.1.yaml similarity index 73% rename from dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.20python3.8.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl1.1.1.yaml index d0ad4c12d2067..58533cf57a553 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.20python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl1.1.1.yaml @@ -1,3 +1,5 @@ +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -12,6 +14,8 @@ cuda_compiler: - nvcc cuda_compiler_version: - '10.2' +cuda_compiler_version_min: +- '10.2' cxx_compiler: - vs2019 gflags: @@ -19,29 +23,35 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libcrc32c: +- '1.1' +libcurl: +- '7' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: - 1.9.3 numpy: +- '1.21' +- '1.23' +- '1.20' - '1.20' openssl: - 1.1.1 pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: +- 3.10.* *_cpython +- 3.11.* *_cpython - 3.8.* *_cpython +- 3.9.* *_cpython re2: - 2022.06.01 snappy: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.21python3.10.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml similarity index 71% rename from dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.21python3.10.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml index a5de05e8241da..0767583485666 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.21python3.10.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml @@ -1,3 +1,5 @@ +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -11,7 +13,9 @@ channel_targets: cuda_compiler: - nvcc cuda_compiler_version: -- None +- '10.2' +cuda_compiler_version_min: +- '10.2' cxx_compiler: - vs2019 gflags: @@ -19,29 +23,35 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libcrc32c: +- '1.1' +libcurl: +- '7' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: - 1.9.3 numpy: - '1.21' +- '1.23' +- '1.20' +- '1.20' openssl: -- 1.1.1 +- '3' pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: - 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython re2: - 2022.06.01 snappy: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.20python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.20python3.7.____cpython.yaml deleted file mode 100644 index 4fcc0f797f204..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.20python3.7.____cpython.yaml +++ /dev/null @@ -1,59 +0,0 @@ -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- vs2019 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cxx_compiler: -- vs2019 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' -libabseil: -- '20220623.0' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.20' -openssl: -- 1.1.1 -pin_run_as_build: - bzip2: - max_pin: x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.7.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- win-64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.20python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.20python3.8.____cpython.yaml deleted file mode 100644 index 97e3a9332e5d8..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.20python3.8.____cpython.yaml +++ /dev/null @@ -1,59 +0,0 @@ -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- vs2019 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cxx_compiler: -- vs2019 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' -libabseil: -- '20220623.0' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.20' -openssl: -- 1.1.1 -pin_run_as_build: - bzip2: - max_pin: x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.8.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- win-64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.20python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.20python3.9.____cpython.yaml deleted file mode 100644 index 9d771ea6d13be..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.20python3.9.____cpython.yaml +++ /dev/null @@ -1,59 +0,0 @@ -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- vs2019 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cxx_compiler: -- vs2019 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' -libabseil: -- '20220623.0' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.20' -openssl: -- 1.1.1 -pin_run_as_build: - bzip2: - max_pin: x - python: - min_pin: x.x - max_pin: x.x - zlib: - max_pin: x.x -python: -- 3.9.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- win-64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.21python3.10.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl1.1.1.yaml similarity index 73% rename from dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.21python3.10.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl1.1.1.yaml index f8b50812c5f62..610426257be04 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.21python3.10.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl1.1.1.yaml @@ -1,3 +1,5 @@ +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -11,6 +13,8 @@ channel_targets: cuda_compiler: - nvcc cuda_compiler_version: +- None +cuda_compiler_version_min: - '10.2' cxx_compiler: - vs2019 @@ -19,29 +23,35 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libcrc32c: +- '1.1' +libcurl: +- '7' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: - 1.9.3 numpy: - '1.21' +- '1.23' +- '1.20' +- '1.20' openssl: - 1.1.1 pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: - 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython re2: - 2022.06.01 snappy: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.20python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml similarity index 72% rename from dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.20python3.9.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml index ad93f736717d0..6430884dac83b 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2numpy1.20python3.9.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml @@ -1,3 +1,5 @@ +aws_sdk_cpp: +- 1.9.379 bzip2: - '1' c_ares: @@ -11,6 +13,8 @@ channel_targets: cuda_compiler: - nvcc cuda_compiler_version: +- None +cuda_compiler_version_min: - '10.2' cxx_compiler: - vs2019 @@ -19,28 +23,34 @@ gflags: glog: - '0.6' google_cloud_cpp: -- '2.2' -grpc_cpp: -- '1.47' +- 2.3.0 libabseil: - '20220623.0' +libcrc32c: +- '1.1' +libcurl: +- '7' +libgrpc: +- '1.49' libprotobuf: - '3.21' lz4_c: - 1.9.3 numpy: +- '1.21' +- '1.23' +- '1.20' - '1.20' openssl: -- 1.1.1 +- '3' pin_run_as_build: - bzip2: - max_pin: x python: min_pin: x.x max_pin: x.x - zlib: - max_pin: x.x python: +- 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython - 3.9.* *_cpython re2: - 2022.06.01 diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 8153aed914106..f544f10cb557a 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -253,8 +253,8 @@ tasks: config: linux_64_c_compiler_version9cuda_compiler_versionNonecxx_compiler_version9numpy1.18python3.7.____cpython r_config: linux_64_r_base4.0 artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 + - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda conda-linux-gcc-py37-cpu-r41: ci: azure @@ -263,70 +263,104 @@ tasks: config: linux_64_c_compiler_version9cuda_compiler_versionNonecxx_compiler_version9numpy1.18python3.7.____cpython r_config: linux_64_r_base4.1 artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 + - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda - conda-linux-gcc-py38-cpu: + conda-linux-gcc-openssl1.1.1-cpu: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_64_c_compiler_version9cuda_compiler_versionNonecxx_compiler_version9numpy1.18python3.8.____cpython + config: linux_64_cuda_compiler_versionNoneopenssl1.1.1 artifacts: - - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-gcc-py39-cpu: + conda-linux-gcc-openssl3-cpu: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_64_c_compiler_version9cuda_compiler_versionNonecxx_compiler_version9numpy1.19python3.9.____cpython + config: linux_64_cuda_compiler_versionNoneopenssl3 artifacts: - - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-gcc-py310-cpu: + conda-linux-gcc-openssl1.1.1-cuda: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_64_c_compiler_version9cuda_compiler_versionNonecxx_compiler_version9numpy1.21python3.10.____cpython + config: linux_64_cuda_compiler_version10.2openssl1.1.1 artifacts: - - arrow-cpp-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2 + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda -{% for python_version, numpy_version in [("3.7", "1.18"), - ("3.8", "1.18"), - ("3.9", "1.19"), - ("3.10", "1.21")] %} - {% set pyver = python_version | replace(".", "") %} + conda-linux-gcc-openssl3-cuda: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_64_cuda_compiler_version10.2openssl3 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda - conda-linux-gcc-py{{ pyver }}-cuda: + conda-linux-gcc-openssl1.1.1-aarch64: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy{{ numpy_version }}python{{ python_version }}.____cpython + config: linux_aarch64_cuda_compiler_versionNoneopenssl1.1.1 artifacts: - - arrow-cpp-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cuda.tar.bz2 - - pyarrow-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cuda.tar.bz2 + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-gcc-py{{ pyver }}-arm64: + conda-linux-gcc-openssl3-aarch64: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_aarch64_numpy{{ numpy_version }}python{{ python_version }}.____cpython + config: linux_aarch64_cuda_compiler_versionNoneopenssl3 artifacts: - - arrow-cpp-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2 + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-gcc-py{{ pyver }}-ppc64le: + conda-linux-gcc-openssl1.1.1-ppc64le: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_ppc64le_numpy{{ numpy_version }}python{{ python_version }}.____cpython + config: linux_ppc64le_cuda_compiler_versionNoneopenssl1.1.1 artifacts: - - arrow-cpp-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2 + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda -{% endfor %} + conda-linux-gcc-openssl3-ppc64le: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_ppc64le_cuda_compiler_versionNoneopenssl3 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda ############################## Conda OSX #################################### @@ -337,8 +371,8 @@ tasks: config: osx_64_numpy1.18python3.7.____cpython r_config: osx_64_r_base4.0 artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 + - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda conda-osx-clang-py37-r41: ci: azure @@ -347,62 +381,56 @@ tasks: config: osx_64_numpy1.18python3.7.____cpython r_config: osx_64_r_base4.1 artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 + - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda - conda-osx-clang-py38: + conda-osx-clang-openssl1.1.1: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_64_numpy1.18python3.8.____cpython + config: osx_64_openssl1.1.1 artifacts: - - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-osx-clang-py39: + conda-osx-clang-openssl3: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_64_numpy1.19python3.9.____cpython + config: osx_64_openssl3 artifacts: - - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-osx-clang-py310: + conda-osx-arm64-clang-openssl1.1.1: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_64_numpy1.21python3.10.____cpython + config: osx_arm64_openssl1.1.1 artifacts: - - arrow-cpp-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2 + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-osx-arm64-clang-py38: + conda-osx-arm64-clang-openssl3: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_arm64_numpy1.19python3.8.____cpython + config: osx_arm64_openssl3 artifacts: - - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-osx-arm64-clang-py39: - ci: azure - template: conda-recipes/azure.osx.yml - params: - config: osx_arm64_numpy1.19python3.9.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - - conda-osx-arm64-clang-py310: - ci: azure - template: conda-recipes/azure.osx.yml - params: - config: osx_arm64_numpy1.21python3.10.____cpython - artifacts: - - arrow-cpp-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2 + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda ############################## Conda Windows ################################ @@ -413,8 +441,8 @@ tasks: config: win_64_cuda_compiler_versionNonenumpy1.18python3.7.____cpython r_config: win_64_r_base4.0 artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 + - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda conda-win-vs2019-py37-r41: ci: azure @@ -423,37 +451,56 @@ tasks: config: win_64_cuda_compiler_versionNonenumpy1.18python3.7.____cpython r_config: win_64_r_base4.0 artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 + - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda - conda-win-vs2019-py38: + conda-win-vs2019-openssl1.1.1: ci: azure template: conda-recipes/azure.win.yml params: - config: win_64_cuda_compiler_versionNonenumpy1.18python3.8.____cpython + config: win_64_cuda_compiler_versionNoneopenssl1.1.1 artifacts: - - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-win-vs2019-py39: + conda-win-vs2019-openssl3: ci: azure template: conda-recipes/azure.win.yml params: - config: win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython + config: win_64_cuda_compiler_versionNoneopenssl3 artifacts: - - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-win-vs2019-py310: + conda-win-vs2019-openssl1.1.1-cuda: ci: azure template: conda-recipes/azure.win.yml params: - config: win_64_cuda_compiler_versionNonenumpy1.21python3.10.____cpython + config: win_64_cuda_compiler_versionNoneopenssl1.1.1 artifacts: - - arrow-cpp-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2 - - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2 + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda -# TODO: Windows CUDA + conda-win-vs2019-openssl3-cuda: + ci: azure + template: conda-recipes/azure.win.yml + params: + config: win_64_cuda_compiler_versionNoneopenssl3 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda {% for python_version, python_tag, abi_tag in [("3.7", "cp37", "cp37m"), ("3.8", "cp38", "cp38"), From 759e7050d452539f58eabdb522028bc58421e6cb Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 5 Dec 2022 15:59:27 +1100 Subject: [PATCH 04/25] avoid patching by moving relevant assertion into gcs_server fixture --- .gitattributes | 2 -- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 4 --- ...all-tests-requiring-gcs_server-fixtu.patch | 36 ------------------- python/pyarrow/tests/conftest.py | 4 ++- python/pyarrow/tests/test_fs.py | 2 -- 5 files changed, 3 insertions(+), 45 deletions(-) delete mode 100644 dev/tasks/conda-recipes/arrow-cpp/patches/0001-definitely-skip-all-tests-requiring-gcs_server-fixtu.patch diff --git a/.gitattributes b/.gitattributes index e91661c424b4f..1a5b156b491fd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,3 @@ -*.patch binary - r/R/RcppExports.R linguist-generated=true r/R/arrowExports.R linguist-generated=true r/src/RcppExports.cpp linguist-generated=true diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 0bb8359c3e235..bd86d907d5bb6 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -12,10 +12,6 @@ package: source: path: ../../../../ - patches: - # switch off tests that require the as-yet unpackaged - # https://github.com/googleapis/storage-testbench - - patches/0001-definitely-skip-all-tests-requiring-gcs_server-fixtu.patch build: number: 0 diff --git a/dev/tasks/conda-recipes/arrow-cpp/patches/0001-definitely-skip-all-tests-requiring-gcs_server-fixtu.patch b/dev/tasks/conda-recipes/arrow-cpp/patches/0001-definitely-skip-all-tests-requiring-gcs_server-fixtu.patch deleted file mode 100644 index 7dd04b616d053..0000000000000 --- a/dev/tasks/conda-recipes/arrow-cpp/patches/0001-definitely-skip-all-tests-requiring-gcs_server-fixtu.patch +++ /dev/null @@ -1,36 +0,0 @@ -From 2549974357216cab621d214582484e9b056c2984 Mon Sep 17 00:00:00 2001 -From: "H. Vetinari" -Date: Wed, 23 Nov 2022 17:10:52 +1100 -Subject: [PATCH] definitely skip all tests requiring gcs_server fixture - -since we don't have https://github.com/googleapis/storage-testbench -in conda-forge so far, which is responsible for the `python -m testbench` ---- - python/pyarrow/tests/conftest.py | 14 +------------- - 1 file changed, 1 insertion(+), 13 deletions(-) - -diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py -index 154ef79dc..d1d688814 100644 ---- a/python/pyarrow/tests/conftest.py -+++ b/python/pyarrow/tests/conftest.py -@@ -183,16 +183,4 @@ def gcs_server(): - env = os.environ.copy() - args = [sys.executable, '-m', 'testbench', '--port', str(port)] - proc = None -- try: -- proc = subprocess.Popen(args, env=env) -- except OSError as e: -- pytest.skip(f"Command {args} failed to execute: {e}") -- else: -- yield { -- 'connection': ('localhost', port), -- 'process': proc, -- } -- finally: -- if proc is not None: -- proc.kill() -- proc.wait() -+ pytest.skip(f"Tests depending on gcs_server disabled for conda builds for now") --- -2.38.1.windows.1 - diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index 154ef79dcd574..83381c982a57e 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -185,7 +185,9 @@ def gcs_server(): proc = None try: proc = subprocess.Popen(args, env=env) - except OSError as e: + # Make sure the server is alive. + assert proc.poll() is None + except (AssertionError, OSError) as e: pytest.skip(f"Command {args} failed to execute: {e}") else: yield { diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index feb9fb685e645..1af57ea9407d9 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -207,8 +207,6 @@ def gcsfs(request, gcs_server): host, port = gcs_server['connection'] bucket = 'pyarrow-filesystem/' - # Make sure the server is alive. - assert gcs_server['process'].poll() is None fs = GcsFileSystem( endpoint_override=f'{host}:{port}', From 85d91c6c427e4d89d0530edac57f8b02a3158782 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 5 Dec 2022 16:03:24 +1100 Subject: [PATCH 05/25] rename conda jobs to get better grouping in crossbow --- dev/tasks/tasks.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index f544f10cb557a..a1b1ddc96f28c 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -266,7 +266,7 @@ tasks: - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda - conda-linux-gcc-openssl1.1.1-cpu: + conda-linux-gcc-cpu-x64-openssl1.1.1: ci: azure template: conda-recipes/azure.linux.yml params: @@ -278,7 +278,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-gcc-openssl3-cpu: + conda-linux-gcc-cpu-x64-openssl3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -290,7 +290,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-gcc-openssl1.1.1-cuda: + conda-linux-gcc-cuda-x64-openssl1.1.1: ci: azure template: conda-recipes/azure.linux.yml params: @@ -302,7 +302,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda - conda-linux-gcc-openssl3-cuda: + conda-linux-gcc-cuda-x64-openssl3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -314,7 +314,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda - conda-linux-gcc-openssl1.1.1-aarch64: + conda-linux-gcc-cpu-aarch64-openssl1.1.1: ci: azure template: conda-recipes/azure.linux.yml params: @@ -326,7 +326,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-gcc-openssl3-aarch64: + conda-linux-gcc-cpu-aarch64-openssl3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -338,7 +338,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-gcc-openssl1.1.1-ppc64le: + conda-linux-gcc-cpu-ppc64le-openssl1.1.1: ci: azure template: conda-recipes/azure.linux.yml params: @@ -350,7 +350,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-gcc-openssl3-ppc64le: + conda-linux-gcc-cpu-ppc64le-openssl3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -454,7 +454,7 @@ tasks: - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda - conda-win-vs2019-openssl1.1.1: + conda-win-vs2019-cpu-openssl1.1.1: ci: azure template: conda-recipes/azure.win.yml params: @@ -466,7 +466,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-win-vs2019-openssl3: + conda-win-vs2019-cpu-openssl3: ci: azure template: conda-recipes/azure.win.yml params: @@ -478,7 +478,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-win-vs2019-openssl1.1.1-cuda: + conda-win-vs2019-cuda-openssl1.1.1: ci: azure template: conda-recipes/azure.win.yml params: @@ -490,7 +490,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda - conda-win-vs2019-openssl3-cuda: + conda-win-vs2019-cuda-openssl3: ci: azure template: conda-recipes/azure.win.yml params: From dae66c1f1e5d80bd940ba53264b0434d876b11e6 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 5 Dec 2022 16:31:05 +1100 Subject: [PATCH 06/25] sync R recipe with feedstock --- dev/tasks/conda-recipes/r-arrow/bld.bat | 5 +++++ dev/tasks/conda-recipes/r-arrow/build.sh | 15 ++++++++++----- dev/tasks/conda-recipes/r-arrow/build_win.sh | 3 +-- dev/tasks/conda-recipes/r-arrow/meta.yaml | 5 ++--- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/dev/tasks/conda-recipes/r-arrow/bld.bat b/dev/tasks/conda-recipes/r-arrow/bld.bat index a193ddc0a7748..b4f84d0dced65 100644 --- a/dev/tasks/conda-recipes/r-arrow/bld.bat +++ b/dev/tasks/conda-recipes/r-arrow/bld.bat @@ -1,9 +1,14 @@ +@echo on + bash %RECIPE_DIR%/build_win.sh IF %ERRORLEVEL% NEQ 0 exit 1 + cp %RECIPE_DIR%/configure.win r IF %ERRORLEVEL% NEQ 0 exit 1 + cp %RECIPE_DIR%/install.libs.R r/src IF %ERRORLEVEL% NEQ 0 exit 1 + set "MAKEFLAGS=-j%CPU_COUNT%" "%R%" CMD INSTALL --build r IF %ERRORLEVEL% NEQ 0 exit 1 diff --git a/dev/tasks/conda-recipes/r-arrow/build.sh b/dev/tasks/conda-recipes/r-arrow/build.sh index 724a8afd4fb2e..5abe54e38f8b3 100644 --- a/dev/tasks/conda-recipes/r-arrow/build.sh +++ b/dev/tasks/conda-recipes/r-arrow/build.sh @@ -1,9 +1,14 @@ -#!/usr/bin/env bash - +#!/bin/bash set -ex export DISABLE_AUTOBREW=1 -# See https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk -export ARROW_R_CXXFLAGS="${ARROW_R_CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" -$R CMD INSTALL --build r/. +# set C++17 due to abseil +export ARROW_R_CXXFLAGS="${ARROW_R_CXXFLAGS} -std=c++17" + +if [[ "${target_platform}" == osx-* ]]; then + # See https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk + export ARROW_R_CXXFLAGS="${ARROW_R_CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" +fi + +${R} CMD INSTALL --build r/. ${R_ARGS} diff --git a/dev/tasks/conda-recipes/r-arrow/build_win.sh b/dev/tasks/conda-recipes/r-arrow/build_win.sh index 22c07d6e056f2..0c9a85ce3943a 100755 --- a/dev/tasks/conda-recipes/r-arrow/build_win.sh +++ b/dev/tasks/conda-recipes/r-arrow/build_win.sh @@ -1,8 +1,7 @@ -#!/usr/bin/env bash +#!/bin/bash set -exuo pipefail - # Rename arrow.dll to lib_arrow.dll to avoid conflicts with the arrow-cpp arrow.dll sed -i -e 's/void R_init_arrow/__declspec(dllexport) void R_init_lib_arrow/g' r/src/arrowExports.cpp sed -i -e 's/useDynLib(arrow/useDynLib(lib_arrow/g' r/NAMESPACE diff --git a/dev/tasks/conda-recipes/r-arrow/meta.yaml b/dev/tasks/conda-recipes/r-arrow/meta.yaml index 1a906f3462859..3d053a781b0fe 100644 --- a/dev/tasks/conda-recipes/r-arrow/meta.yaml +++ b/dev/tasks/conda-recipes/r-arrow/meta.yaml @@ -11,7 +11,7 @@ source: build: merge_build_host: true # [win] - number: 1 + number: 0 rpaths: - lib/R/lib/ - lib/ @@ -25,7 +25,6 @@ requirements: - r-purrr # [build_platform != target_platform] - r-rlang # [build_platform != target_platform] - r-tidyselect # [build_platform != target_platform] - - python 3.9 # [build_platform != target_platform] - {{ compiler('c') }} # [not win] - {{ compiler('cxx') }} # [not win] - {{ compiler('r_clang') }} # [win] @@ -38,8 +37,8 @@ requirements: host: # Needs to be here, otherwise merge_build_host runs into issues - pkg-config # [win] + - libarrow {{ version }} - r-base - - arrow-cpp {{ version }} - r-r6 - r-cpp11 - r-assertthat From 115fe7573902adb30f60a1b39c2eaa3759fdb349 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 5 Dec 2022 16:32:10 +1100 Subject: [PATCH 07/25] renormalize line-endings in dev/tasks/conda-recipes/r-arrow done by: `git add --renormalize -- dev/tasks/conda-recipes/r-arrow` --- dev/tasks/conda-recipes/r-arrow/bld.bat | 28 ++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/dev/tasks/conda-recipes/r-arrow/bld.bat b/dev/tasks/conda-recipes/r-arrow/bld.bat index b4f84d0dced65..04d59f282d84b 100644 --- a/dev/tasks/conda-recipes/r-arrow/bld.bat +++ b/dev/tasks/conda-recipes/r-arrow/bld.bat @@ -1,14 +1,14 @@ -@echo on - -bash %RECIPE_DIR%/build_win.sh -IF %ERRORLEVEL% NEQ 0 exit 1 - -cp %RECIPE_DIR%/configure.win r -IF %ERRORLEVEL% NEQ 0 exit 1 - -cp %RECIPE_DIR%/install.libs.R r/src -IF %ERRORLEVEL% NEQ 0 exit 1 - -set "MAKEFLAGS=-j%CPU_COUNT%" -"%R%" CMD INSTALL --build r -IF %ERRORLEVEL% NEQ 0 exit 1 +@echo on + +bash %RECIPE_DIR%/build_win.sh +IF %ERRORLEVEL% NEQ 0 exit 1 + +cp %RECIPE_DIR%/configure.win r +IF %ERRORLEVEL% NEQ 0 exit 1 + +cp %RECIPE_DIR%/install.libs.R r/src +IF %ERRORLEVEL% NEQ 0 exit 1 + +set "MAKEFLAGS=-j%CPU_COUNT%" +"%R%" CMD INSTALL --build r +IF %ERRORLEVEL% NEQ 0 exit 1 From 7e2155e6ac2af130f8158f551e98caade53da47e Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 5 Dec 2022 17:02:53 +1100 Subject: [PATCH 08/25] update conda-jobs for R --- .../.ci_support/r/linux_64_r_base4.1.yaml | 4 +- ...r_base4.0.yaml => linux_64_r_base4.2.yaml} | 6 +- .../r/linux_aarch64_r_base4.1.yaml | 31 +++++ .../r/linux_aarch64_r_base4.2.yaml | 31 +++++ .../.ci_support/r/osx_64_r_base4.1.yaml | 4 +- ...4_r_base4.0.yaml => osx_64_r_base4.2.yaml} | 6 +- .../.ci_support/r/osx_arm64_r_base4.1.yaml | 27 +++++ .../.ci_support/r/osx_arm64_r_base4.2.yaml | 27 +++++ .../r/{win_64_r_base4.1.yaml => win_64_.yaml} | 0 .../.ci_support/r/win_64_r_base4.0.yaml | 12 -- dev/tasks/tasks.yml | 113 ++++++++++++------ 11 files changed, 200 insertions(+), 61 deletions(-) rename dev/tasks/conda-recipes/.ci_support/r/{linux_64_r_base4.0.yaml => linux_64_r_base4.2.yaml} (94%) create mode 100644 dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml create mode 100644 dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml rename dev/tasks/conda-recipes/.ci_support/r/{osx_64_r_base4.0.yaml => osx_64_r_base4.2.yaml} (94%) create mode 100644 dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml create mode 100644 dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.2.yaml rename dev/tasks/conda-recipes/.ci_support/r/{win_64_r_base4.1.yaml => win_64_.yaml} (100%) delete mode 100644 dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml index 00d944cb8616c..117a09f57490f 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml @@ -1,7 +1,7 @@ c_compiler: - gcc c_compiler_version: -- '9' +- '10' cdt_name: - cos6 channel_sources: @@ -11,7 +11,7 @@ channel_targets: cxx_compiler: - gxx cxx_compiler_version: -- '9' +- '10' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 pin_run_as_build: diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml similarity index 94% rename from dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml rename to dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml index bb4dbcbbe633a..bab275a30d56b 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml @@ -1,7 +1,7 @@ c_compiler: - gcc c_compiler_version: -- '9' +- '10' cdt_name: - cos6 channel_sources: @@ -11,7 +11,7 @@ channel_targets: cxx_compiler: - gxx cxx_compiler_version: -- '9' +- '10' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 pin_run_as_build: @@ -19,7 +19,7 @@ pin_run_as_build: min_pin: x.x max_pin: x.x r_base: -- '4.0' +- '4.2' target_platform: - linux-64 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml new file mode 100644 index 0000000000000..ad96522d73ba6 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml @@ -0,0 +1,31 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +c_compiler: +- gcc +c_compiler_version: +- '10' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- gxx +cxx_compiler_version: +- '10' +docker_image: +- quay.io/condaforge/linux-anvil-cos7-x86_64 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.1' +target_platform: +- linux-aarch64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml new file mode 100644 index 0000000000000..f92562e13640b --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml @@ -0,0 +1,31 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +c_compiler: +- gcc +c_compiler_version: +- '10' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- gxx +cxx_compiler_version: +- '10' +docker_image: +- quay.io/condaforge/linux-anvil-cos7-x86_64 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.2' +target_platform: +- linux-aarch64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml index 8926fd82b2d0b..7e5b3d08039c1 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml @@ -3,7 +3,7 @@ MACOSX_DEPLOYMENT_TARGET: c_compiler: - clang c_compiler_version: -- '11' +- '14' channel_sources: - conda-forge channel_targets: @@ -11,7 +11,7 @@ channel_targets: cxx_compiler: - clangxx cxx_compiler_version: -- '11' +- '14' macos_machine: - x86_64-apple-darwin13.4.0 pin_run_as_build: diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.2.yaml similarity index 94% rename from dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml rename to dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.2.yaml index 20f3879b30a5d..25437ee4adcfe 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.2.yaml @@ -3,7 +3,7 @@ MACOSX_DEPLOYMENT_TARGET: c_compiler: - clang c_compiler_version: -- '11' +- '14' channel_sources: - conda-forge channel_targets: @@ -11,7 +11,7 @@ channel_targets: cxx_compiler: - clangxx cxx_compiler_version: -- '11' +- '14' macos_machine: - x86_64-apple-darwin13.4.0 pin_run_as_build: @@ -19,7 +19,7 @@ pin_run_as_build: min_pin: x.x max_pin: x.x r_base: -- '4.0' +- '4.2' target_platform: - osx-64 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml new file mode 100644 index 0000000000000..5c58110332f42 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml @@ -0,0 +1,27 @@ +MACOSX_DEPLOYMENT_TARGET: +- '11.0' +c_compiler: +- clang +c_compiler_version: +- '14' +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- clangxx +cxx_compiler_version: +- '14' +macos_machine: +- arm64-apple-darwin20.0.0 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.1' +target_platform: +- osx-arm64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.2.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.2.yaml new file mode 100644 index 0000000000000..1557b23ff96af --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.2.yaml @@ -0,0 +1,27 @@ +MACOSX_DEPLOYMENT_TARGET: +- '11.0' +c_compiler: +- clang +c_compiler_version: +- '14' +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- clangxx +cxx_compiler_version: +- '14' +macos_machine: +- arm64-apple-darwin20.0.0 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.2' +target_platform: +- osx-arm64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/win_64_.yaml similarity index 100% rename from dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.1.yaml rename to dev/tasks/conda-recipes/.ci_support/r/win_64_.yaml diff --git a/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml b/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml deleted file mode 100644 index bb8f97216368d..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml +++ /dev/null @@ -1,12 +0,0 @@ -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -pin_run_as_build: - r-base: - min_pin: x.x - max_pin: x.x -r_base: -- '4.0' -target_platform: -- win-64 diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index a1b1ddc96f28c..a81f826a87bbd 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -228,7 +228,7 @@ tasks: image: minimal-{{ kind }} {% endfor %} - ############################## Conda Linux ############################ + ############################# Conda Linux (x64) ############################# conda-clean: ci: azure @@ -240,31 +240,30 @@ tasks: # the same feedstock as the dependency matrix is the same for them as # Python and the OS are the main dimension. The R package `r-arrow` is # an independent feedstock as it doesn't have the Python but the - # R dimension. To limit the number of CI jobs, we are building `r-arrow` - # with the Python 3.7 jobs. + # R dimension. # * The files in `dev/tasks/conda-recipes/.ci_support/` are automatically # generated and to be synced regularly from the feedstock. We have no way # yet to generate them inside the arrow repository automatically. - conda-linux-gcc-py37-cpu-r40: + conda-linux-gcc-cpu-x64-r41: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_64_c_compiler_version9cuda_compiler_versionNonecxx_compiler_version9numpy1.18python3.7.____cpython - r_config: linux_64_r_base4.0 + config: linux_64_cuda_compiler_versionNoneopenssl3 + r_config: linux_64_r_base4.1 artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-linux-gcc-py37-cpu-r41: + conda-linux-gcc-cpu-x64-r42: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_64_c_compiler_version9cuda_compiler_versionNonecxx_compiler_version9numpy1.18python3.7.____cpython - r_config: linux_64_r_base4.1 + config: linux_64_cuda_compiler_versionNoneopenssl3 + r_config: linux_64_r_base4.2 artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda conda-linux-gcc-cpu-x64-openssl1.1.1: ci: azure @@ -314,6 +313,28 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda + ########################### Conda Linux (aarch64) ########################### + + conda-linux-gcc-cpu-aarch64-r41: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_aarch64_cuda_compiler_versionNoneopenssl3 + r_config: linux_aarch64_r_base4.1 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + + conda-linux-gcc-cpu-aarch64-r42: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_aarch64_cuda_compiler_versionNoneopenssl3 + r_config: linux_aarch64_r_base4.2 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda + conda-linux-gcc-cpu-aarch64-openssl1.1.1: ci: azure template: conda-recipes/azure.linux.yml @@ -338,6 +359,8 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda + ########################### Conda Linux (ppc64le) ########################### + conda-linux-gcc-cpu-ppc64le-openssl1.1.1: ci: azure template: conda-recipes/azure.linux.yml @@ -362,27 +385,27 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - ############################## Conda OSX #################################### + ############################## Conda OSX (x64) ############################## - conda-osx-clang-py37-r40: + conda-osx-clang-r41: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_64_numpy1.18python3.7.____cpython - r_config: osx_64_r_base4.0 + config: osx_64_openssl3 + r_config: osx_64_r_base4.1 artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-osx-clang-py37-r41: + conda-osx-clang-r42: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_64_numpy1.18python3.7.____cpython - r_config: osx_64_r_base4.1 + config: osx_64_openssl3 + r_config: osx_64_r_base4.2 artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda conda-osx-clang-openssl1.1.1: ci: azure @@ -408,6 +431,28 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda + ############################# Conda OSX (arm64) ############################# + + conda-osx-arm64-clang-r41: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: osx_arm64_openssl3 + r_config: osx_arm64_r_base4.1 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + + conda-osx-arm64-clang-r42: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: osx_arm64_openssl3 + r_config: osx_arm64_r_base4.2 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda + conda-osx-arm64-clang-openssl1.1.1: ci: azure template: conda-recipes/azure.osx.yml @@ -434,25 +479,15 @@ tasks: ############################## Conda Windows ################################ - conda-win-vs2019-py37-r40: - ci: azure - template: conda-recipes/azure.win.yml - params: - config: win_64_cuda_compiler_versionNonenumpy1.18python3.7.____cpython - r_config: win_64_r_base4.0 - artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda - - conda-win-vs2019-py37-r41: + conda-win-vs2019-cpu-r41: ci: azure template: conda-recipes/azure.win.yml params: - config: win_64_cuda_compiler_versionNonenumpy1.18python3.7.____cpython - r_config: win_64_r_base4.0 + config: win_64_cuda_compiler_versionNoneopenssl3 + r_config: win_64_ artifacts: - - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.conda + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda conda-win-vs2019-cpu-openssl1.1.1: ci: azure From 39587c3e09a2b80ae6acdcf0f1464932127efc76 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 5 Dec 2022 17:06:07 +1100 Subject: [PATCH 09/25] consistently sort conda tasks: `platform-arch-{cpu,cuda}-` compilers are redundant as they are fixed per platform --- dev/tasks/tasks.yml | 58 +++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index a81f826a87bbd..ccf770f765430 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -85,9 +85,11 @@ groups: - homebrew-r-autobrew - homebrew-r-brew # r-conda tasks - - conda-linux-gcc-py*-cpu-r* - - conda-osx-clang-py*-r* - - conda-win-vs2019-py*-r* + - conda-linux-x64-cpu-r* + - conda-linux-aarch64-cpu-r* + - conda-osx-x64-cpu-r* + - conda-osx-arm64-cpu-r* + - conda-win-x64-cpu-r* - r-binary-packages ruby: @@ -245,7 +247,7 @@ tasks: # generated and to be synced regularly from the feedstock. We have no way # yet to generate them inside the arrow repository automatically. - conda-linux-gcc-cpu-x64-r41: + conda-linux-x64-cpu-r41: ci: azure template: conda-recipes/azure.linux.yml params: @@ -255,7 +257,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-linux-gcc-cpu-x64-r42: + conda-linux-x64-cpu-r42: ci: azure template: conda-recipes/azure.linux.yml params: @@ -265,7 +267,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda - conda-linux-gcc-cpu-x64-openssl1.1.1: + conda-linux-x64-cpu-openssl1.1.1: ci: azure template: conda-recipes/azure.linux.yml params: @@ -277,7 +279,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-gcc-cpu-x64-openssl3: + conda-linux-x64-cpu-openssl3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -289,7 +291,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-gcc-cuda-x64-openssl1.1.1: + conda-linux-x64-cuda-openssl1.1.1: ci: azure template: conda-recipes/azure.linux.yml params: @@ -301,7 +303,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda - conda-linux-gcc-cuda-x64-openssl3: + conda-linux-x64-cuda-openssl3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -315,7 +317,7 @@ tasks: ########################### Conda Linux (aarch64) ########################### - conda-linux-gcc-cpu-aarch64-r41: + conda-linux-aarch64-cpu-r41: ci: azure template: conda-recipes/azure.linux.yml params: @@ -325,7 +327,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-linux-gcc-cpu-aarch64-r42: + conda-linux-aarch64-cpu-r42: ci: azure template: conda-recipes/azure.linux.yml params: @@ -335,7 +337,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda - conda-linux-gcc-cpu-aarch64-openssl1.1.1: + conda-linux-aarch64-cpu-openssl1.1.1: ci: azure template: conda-recipes/azure.linux.yml params: @@ -347,7 +349,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-gcc-cpu-aarch64-openssl3: + conda-linux-aarch64-cpu-openssl3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -361,7 +363,7 @@ tasks: ########################### Conda Linux (ppc64le) ########################### - conda-linux-gcc-cpu-ppc64le-openssl1.1.1: + conda-linux-ppc64le-cpu-openssl1.1.1: ci: azure template: conda-recipes/azure.linux.yml params: @@ -373,7 +375,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-gcc-cpu-ppc64le-openssl3: + conda-linux-ppc64le-cpu-openssl3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -387,7 +389,7 @@ tasks: ############################## Conda OSX (x64) ############################## - conda-osx-clang-r41: + conda-osx-x64-cpu-r41: ci: azure template: conda-recipes/azure.osx.yml params: @@ -397,7 +399,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-osx-clang-r42: + conda-osx-x64-cpu-r42: ci: azure template: conda-recipes/azure.osx.yml params: @@ -407,7 +409,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda - conda-osx-clang-openssl1.1.1: + conda-osx-x64-openssl1.1.1: ci: azure template: conda-recipes/azure.osx.yml params: @@ -419,7 +421,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-osx-clang-openssl3: + conda-osx-x64-openssl3: ci: azure template: conda-recipes/azure.osx.yml params: @@ -433,7 +435,7 @@ tasks: ############################# Conda OSX (arm64) ############################# - conda-osx-arm64-clang-r41: + conda-osx-arm64-cpu-r41: ci: azure template: conda-recipes/azure.linux.yml params: @@ -443,7 +445,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-osx-arm64-clang-r42: + conda-osx-arm64-cpu-r42: ci: azure template: conda-recipes/azure.linux.yml params: @@ -453,7 +455,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda - conda-osx-arm64-clang-openssl1.1.1: + conda-osx-arm64-openssl1.1.1: ci: azure template: conda-recipes/azure.osx.yml params: @@ -465,7 +467,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-osx-arm64-clang-openssl3: + conda-osx-arm64-openssl3: ci: azure template: conda-recipes/azure.osx.yml params: @@ -479,7 +481,7 @@ tasks: ############################## Conda Windows ################################ - conda-win-vs2019-cpu-r41: + conda-win-x64-cpu-r41: ci: azure template: conda-recipes/azure.win.yml params: @@ -489,7 +491,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-win-vs2019-cpu-openssl1.1.1: + conda-win-x64-cpu-openssl1.1.1: ci: azure template: conda-recipes/azure.win.yml params: @@ -501,7 +503,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-win-vs2019-cpu-openssl3: + conda-win-x64-cpu-openssl3: ci: azure template: conda-recipes/azure.win.yml params: @@ -513,7 +515,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-win-vs2019-cuda-openssl1.1.1: + conda-win-x64-cuda-openssl1.1.1: ci: azure template: conda-recipes/azure.win.yml params: @@ -525,7 +527,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda - conda-win-vs2019-cuda-openssl3: + conda-win-x64-cuda-openssl3: ci: azure template: conda-recipes/azure.win.yml params: From 8cf566d23ba644a2d9c6e7ca0568085a1f5e67a9 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 5 Dec 2022 18:09:50 +1100 Subject: [PATCH 10/25] distinguish r-jobs from python-jobs more clearly --- dev/tasks/tasks.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index ccf770f765430..50ed65c7456c2 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -267,7 +267,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda - conda-linux-x64-cpu-openssl1.1.1: + conda-linux-x64-cpu-py3-openssl1.1.1: ci: azure template: conda-recipes/azure.linux.yml params: @@ -279,7 +279,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-x64-cpu-openssl3: + conda-linux-x64-cpu-py3-openssl3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -291,7 +291,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-x64-cuda-openssl1.1.1: + conda-linux-x64-cuda-py3-openssl1.1.1: ci: azure template: conda-recipes/azure.linux.yml params: @@ -303,7 +303,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda - conda-linux-x64-cuda-openssl3: + conda-linux-x64-cuda-py3-openssl3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -337,7 +337,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda - conda-linux-aarch64-cpu-openssl1.1.1: + conda-linux-aarch64-cpu-py3-openssl1.1.1: ci: azure template: conda-recipes/azure.linux.yml params: @@ -349,7 +349,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-aarch64-cpu-openssl3: + conda-linux-aarch64-cpu-py3-openssl3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -363,7 +363,7 @@ tasks: ########################### Conda Linux (ppc64le) ########################### - conda-linux-ppc64le-cpu-openssl1.1.1: + conda-linux-ppc64le-cpu-py3-openssl1.1.1: ci: azure template: conda-recipes/azure.linux.yml params: @@ -375,7 +375,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-ppc64le-cpu-openssl3: + conda-linux-ppc64le-cpu-py3-openssl3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -409,7 +409,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda - conda-osx-x64-openssl1.1.1: + conda-osx-x64-cpu-py3-openssl1.1.1: ci: azure template: conda-recipes/azure.osx.yml params: @@ -421,7 +421,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-osx-x64-openssl3: + conda-osx-x64-cpu-py3-openssl3: ci: azure template: conda-recipes/azure.osx.yml params: @@ -455,7 +455,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda - conda-osx-arm64-openssl1.1.1: + conda-osx-arm64-cpu-py3-openssl1.1.1: ci: azure template: conda-recipes/azure.osx.yml params: @@ -467,7 +467,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-osx-arm64-openssl3: + conda-osx-arm64-cpu-py3-openssl3: ci: azure template: conda-recipes/azure.osx.yml params: @@ -491,7 +491,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-win-x64-cpu-openssl1.1.1: + conda-win-x64-cpu-py3-openssl1.1.1: ci: azure template: conda-recipes/azure.win.yml params: @@ -503,7 +503,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-win-x64-cpu-openssl3: + conda-win-x64-cpu-py3-openssl3: ci: azure template: conda-recipes/azure.win.yml params: @@ -515,7 +515,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-win-x64-cuda-openssl1.1.1: + conda-win-x64-cuda-py3-openssl1.1.1: ci: azure template: conda-recipes/azure.win.yml params: @@ -527,7 +527,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda - conda-win-x64-cuda-openssl3: + conda-win-x64-cuda-py3-openssl3: ci: azure template: conda-recipes/azure.win.yml params: From 16f519b37da751f70ed00c70792b6e1f58bcc569 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 5 Dec 2022 18:15:13 +1100 Subject: [PATCH 11/25] update conda CI configs to latest status --- .../linux_64_cuda_compiler_version10.2openssl1.1.1.yaml | 2 +- .../linux_64_cuda_compiler_version10.2openssl3.yaml | 2 +- .../linux_64_cuda_compiler_versionNoneopenssl1.1.1.yaml | 6 +++--- .../linux_64_cuda_compiler_versionNoneopenssl3.yaml | 6 +++--- ...linux_aarch64_cuda_compiler_versionNoneopenssl1.1.1.yaml | 6 +++--- .../linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml | 6 +++--- ...linux_ppc64le_cuda_compiler_versionNoneopenssl1.1.1.yaml | 6 +++--- .../linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml | 6 +++--- .../conda-recipes/.ci_support/osx_64_openssl1.1.1.yaml | 2 +- dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml | 2 +- .../conda-recipes/.ci_support/osx_arm64_openssl1.1.1.yaml | 2 +- dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml | 2 +- .../conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml | 4 ++-- .../conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml | 4 ++-- .../.ci_support/r/linux_aarch64_r_base4.1.yaml | 4 ++-- .../.ci_support/r/linux_aarch64_r_base4.2.yaml | 4 ++-- .../win_64_cuda_compiler_version10.2openssl1.1.1.yaml | 2 +- .../win_64_cuda_compiler_version10.2openssl3.yaml | 2 +- .../win_64_cuda_compiler_versionNoneopenssl1.1.1.yaml | 2 +- .../win_64_cuda_compiler_versionNoneopenssl3.yaml | 2 +- 20 files changed, 36 insertions(+), 36 deletions(-) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl1.1.1.yaml index 09e8dfb7372b3..7ced3c2f0cc39 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl1.1.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl1.1.1.yaml @@ -31,7 +31,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libgrpc: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml index 99022e84887e8..5153ba8d32c9d 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml @@ -31,7 +31,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libgrpc: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl1.1.1.yaml index edba301f4a1f0..2b57757cbf903 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl1.1.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl1.1.1.yaml @@ -7,7 +7,7 @@ c_ares: c_compiler: - gcc c_compiler_version: -- '10' +- '11' cdt_name: - cos6 channel_sources: @@ -23,7 +23,7 @@ cuda_compiler_version_min: cxx_compiler: - gxx cxx_compiler_version: -- '10' +- '11' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 gflags: @@ -31,7 +31,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libgrpc: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml index 120d8eb622b57..e22596823b30a 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml @@ -7,7 +7,7 @@ c_ares: c_compiler: - gcc c_compiler_version: -- '10' +- '11' cdt_name: - cos6 channel_sources: @@ -23,7 +23,7 @@ cuda_compiler_version_min: cxx_compiler: - gxx cxx_compiler_version: -- '10' +- '11' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 gflags: @@ -31,7 +31,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libgrpc: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl1.1.1.yaml index 7ec61b35a0f59..43bacb534a425 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl1.1.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl1.1.1.yaml @@ -9,7 +9,7 @@ c_ares: c_compiler: - gcc c_compiler_version: -- '10' +- '11' cdt_arch: - aarch64 cdt_name: @@ -23,7 +23,7 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '10' +- '11' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 gflags: @@ -31,7 +31,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libgrpc: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml index fa91ba903dec6..a07a0762efb33 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml @@ -9,7 +9,7 @@ c_ares: c_compiler: - gcc c_compiler_version: -- '10' +- '11' cdt_arch: - aarch64 cdt_name: @@ -23,7 +23,7 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '10' +- '11' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 gflags: @@ -31,7 +31,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libgrpc: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl1.1.1.yaml index c5a546f91195e..b9a211946d23a 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl1.1.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl1.1.1.yaml @@ -7,7 +7,7 @@ c_ares: c_compiler: - gcc c_compiler_version: -- '10' +- '11' cdt_name: - cos7 channel_sources: @@ -19,7 +19,7 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '10' +- '11' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 gflags: @@ -27,7 +27,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libgrpc: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml index 16c5692757d17..25c42f8ccf06b 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml @@ -7,7 +7,7 @@ c_ares: c_compiler: - gcc c_compiler_version: -- '10' +- '11' cdt_name: - cos7 channel_sources: @@ -19,7 +19,7 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '10' +- '11' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 gflags: @@ -27,7 +27,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libgrpc: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_openssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_openssl1.1.1.yaml index a52bf9bbd3558..5ef81299bb256 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_openssl1.1.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_openssl1.1.1.yaml @@ -25,7 +25,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libgrpc: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml index f47c333a5b2b1..1408d2f0e6d11 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml @@ -25,7 +25,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libgrpc: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl1.1.1.yaml index 59128de87d6c4..7bcf2452987ba 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl1.1.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl1.1.1.yaml @@ -25,7 +25,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libgrpc: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml index 4c08c312f839c..56f28d01bcf98 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml @@ -25,7 +25,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libgrpc: diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml index 117a09f57490f..96784cc8b6f05 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml @@ -1,7 +1,7 @@ c_compiler: - gcc c_compiler_version: -- '10' +- '11' cdt_name: - cos6 channel_sources: @@ -11,7 +11,7 @@ channel_targets: cxx_compiler: - gxx cxx_compiler_version: -- '10' +- '11' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 pin_run_as_build: diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml index bab275a30d56b..38753baa7ed09 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml @@ -1,7 +1,7 @@ c_compiler: - gcc c_compiler_version: -- '10' +- '11' cdt_name: - cos6 channel_sources: @@ -11,7 +11,7 @@ channel_targets: cxx_compiler: - gxx cxx_compiler_version: -- '10' +- '11' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 pin_run_as_build: diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml index ad96522d73ba6..413d8c6583fd4 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml @@ -3,7 +3,7 @@ BUILD: c_compiler: - gcc c_compiler_version: -- '10' +- '11' cdt_arch: - aarch64 cdt_name: @@ -15,7 +15,7 @@ channel_targets: cxx_compiler: - gxx cxx_compiler_version: -- '10' +- '11' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 pin_run_as_build: diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml index f92562e13640b..2913bbb4f141f 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml @@ -3,7 +3,7 @@ BUILD: c_compiler: - gcc c_compiler_version: -- '10' +- '11' cdt_arch: - aarch64 cdt_name: @@ -15,7 +15,7 @@ channel_targets: cxx_compiler: - gxx cxx_compiler_version: -- '10' +- '11' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 pin_run_as_build: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl1.1.1.yaml index 58533cf57a553..19ff60bd25297 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl1.1.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl1.1.1.yaml @@ -23,7 +23,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libcrc32c: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml index 0767583485666..66f031b56f035 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml @@ -23,7 +23,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libcrc32c: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl1.1.1.yaml index 610426257be04..63a3d36c1dbbd 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl1.1.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl1.1.1.yaml @@ -23,7 +23,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libcrc32c: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml index 6430884dac83b..b18fc222a8004 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml @@ -23,7 +23,7 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.3.0 +- 2.5.0 libabseil: - '20220623.0' libcrc32c: From c39d2081c25078551f10c028c8d95da0992b3dd3 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 5 Dec 2022 18:33:41 +1100 Subject: [PATCH 12/25] only run pyarrow tests for one python version this is especially relevant for the emulated tests on linux-aarch64 --- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index bd86d907d5bb6..3d21004d531cc 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -381,7 +381,10 @@ outputs: {% set tests_to_skip = tests_to_skip + " or (test_memory and test_env_var)" %} # [unix] # test is broken; header is in $PREFIX, not $SP_DIR {% set tests_to_skip = tests_to_skip + " or (test_misc and test_get_include)" %} # [unix] - - pytest -v -rfEs -k "not ({{ tests_to_skip }})" # [not ppc64le] + # ^^^^^^^ TESTS THAT SHOULDN'T HAVE TO BE SKIPPED ^^^^^^^ + + # crossbow CI: enough to run for one python version + - pytest -v -rfEs -k "not ({{ tests_to_skip }})" # [(not ppc64le) and py==310] # NOTE: test suite is skipped on ppc due to bugs in QEMU code that cause # CI to fail, even though the tests would run through on native hardware From d8462a28d2527d712c26ce1aa4eea3bdc2b9325d Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 5 Dec 2022 20:36:46 +1100 Subject: [PATCH 13/25] only run conda builds against openssl 3.0 --- ...cuda_compiler_version10.2openssl1.1.1.yaml | 80 ------------- ...cuda_compiler_versionNoneopenssl1.1.1.yaml | 80 ------------- ...cuda_compiler_versionNoneopenssl1.1.1.yaml | 80 ------------- ...cuda_compiler_versionNoneopenssl1.1.1.yaml | 76 ------------ .../.ci_support/osx_64_openssl1.1.1.yaml | 73 ------------ .../.ci_support/osx_arm64_openssl1.1.1.yaml | 73 ------------ ...cuda_compiler_version10.2openssl1.1.1.yaml | 69 ----------- ...cuda_compiler_versionNoneopenssl1.1.1.yaml | 69 ----------- dev/tasks/tasks.yml | 112 ++---------------- 9 files changed, 8 insertions(+), 704 deletions(-) delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl1.1.1.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl1.1.1.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl1.1.1.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl1.1.1.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/osx_64_openssl1.1.1.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl1.1.1.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl1.1.1.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl1.1.1.yaml diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl1.1.1.yaml deleted file mode 100644 index 7ced3c2f0cc39..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl1.1.1.yaml +++ /dev/null @@ -1,80 +0,0 @@ -aws_sdk_cpp: -- 1.9.379 -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '7' -cdt_name: -- cos6 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.2' -cuda_compiler_version_min: -- '10.2' -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-cuda:10.2 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- 2.5.0 -libabseil: -- '20220623.0' -libgrpc: -- '1.49' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.21' -- '1.23' -- '1.20' -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.8.0 -pin_run_as_build: - python: - min_pin: x.x - max_pin: x.x -python: -- 3.10.* *_cpython -- 3.11.* *_cpython -- 3.8.* *_cpython -- 3.9.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- linux-64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version - - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl1.1.1.yaml deleted file mode 100644 index 2b57757cbf903..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl1.1.1.yaml +++ /dev/null @@ -1,80 +0,0 @@ -aws_sdk_cpp: -- 1.9.379 -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '11' -cdt_name: -- cos6 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cuda_compiler_version_min: -- '10.2' -cxx_compiler: -- gxx -cxx_compiler_version: -- '11' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- 2.5.0 -libabseil: -- '20220623.0' -libgrpc: -- '1.49' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.21' -- '1.23' -- '1.20' -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.8.0 -pin_run_as_build: - python: - min_pin: x.x - max_pin: x.x -python: -- 3.10.* *_cpython -- 3.11.* *_cpython -- 3.8.* *_cpython -- 3.9.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- linux-64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version - - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl1.1.1.yaml deleted file mode 100644 index 43bacb534a425..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl1.1.1.yaml +++ /dev/null @@ -1,80 +0,0 @@ -BUILD: -- aarch64-conda_cos7-linux-gnu -aws_sdk_cpp: -- 1.9.379 -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '11' -cdt_arch: -- aarch64 -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '11' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- 2.5.0 -libabseil: -- '20220623.0' -libgrpc: -- '1.49' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.21' -- '1.23' -- '1.20' -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.8.0 -pin_run_as_build: - python: - min_pin: x.x - max_pin: x.x -python: -- 3.10.* *_cpython -- 3.11.* *_cpython -- 3.8.* *_cpython -- 3.9.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- linux-aarch64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version - - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl1.1.1.yaml deleted file mode 100644 index b9a211946d23a..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl1.1.1.yaml +++ /dev/null @@ -1,76 +0,0 @@ -aws_sdk_cpp: -- 1.9.379 -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- gcc -c_compiler_version: -- '11' -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- gxx -cxx_compiler_version: -- '11' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- 2.5.0 -libabseil: -- '20220623.0' -libgrpc: -- '1.49' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.21' -- '1.23' -- '1.20' -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.8.0 -pin_run_as_build: - python: - min_pin: x.x - max_pin: x.x -python: -- 3.10.* *_cpython -- 3.11.* *_cpython -- 3.8.* *_cpython -- 3.9.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- linux-ppc64le -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version - - cuda_compiler_version - - cdt_name - - docker_image -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_openssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_openssl1.1.1.yaml deleted file mode 100644 index 5ef81299bb256..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_openssl1.1.1.yaml +++ /dev/null @@ -1,73 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '10.9' -aws_sdk_cpp: -- 1.9.379 -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- clang -c_compiler_version: -- '14' -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- clangxx -cxx_compiler_version: -- '14' -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- 2.5.0 -libabseil: -- '20220623.0' -libgrpc: -- '1.49' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -macos_machine: -- x86_64-apple-darwin13.4.0 -numpy: -- '1.21' -- '1.23' -- '1.20' -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.8.0 -pin_run_as_build: - python: - min_pin: x.x - max_pin: x.x -python: -- 3.10.* *_cpython -- 3.11.* *_cpython -- 3.8.* *_cpython -- 3.9.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- osx-64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl1.1.1.yaml deleted file mode 100644 index 7bcf2452987ba..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl1.1.1.yaml +++ /dev/null @@ -1,73 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '11.0' -aws_sdk_cpp: -- 1.9.379 -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- clang -c_compiler_version: -- '14' -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler_version: -- None -cxx_compiler: -- clangxx -cxx_compiler_version: -- '14' -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- 2.5.0 -libabseil: -- '20220623.0' -libgrpc: -- '1.49' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -macos_machine: -- arm64-apple-darwin20.0.0 -numpy: -- '1.21' -- '1.23' -- '1.20' -- '1.20' -openssl: -- 1.1.1 -orc: -- 1.8.0 -pin_run_as_build: - python: - min_pin: x.x - max_pin: x.x -python: -- 3.10.* *_cpython -- 3.11.* *_cpython -- 3.8.* *_cpython -- 3.9.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- osx-arm64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - c_compiler_version - - cxx_compiler_version -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl1.1.1.yaml deleted file mode 100644 index 19ff60bd25297..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl1.1.1.yaml +++ /dev/null @@ -1,69 +0,0 @@ -aws_sdk_cpp: -- 1.9.379 -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- vs2019 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.2' -cuda_compiler_version_min: -- '10.2' -cxx_compiler: -- vs2019 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- 2.5.0 -libabseil: -- '20220623.0' -libcrc32c: -- '1.1' -libcurl: -- '7' -libgrpc: -- '1.49' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.21' -- '1.23' -- '1.20' -- '1.20' -openssl: -- 1.1.1 -pin_run_as_build: - python: - min_pin: x.x - max_pin: x.x -python: -- 3.10.* *_cpython -- 3.11.* *_cpython -- 3.8.* *_cpython -- 3.9.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- win-64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl1.1.1.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl1.1.1.yaml deleted file mode 100644 index 63a3d36c1dbbd..0000000000000 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl1.1.1.yaml +++ /dev/null @@ -1,69 +0,0 @@ -aws_sdk_cpp: -- 1.9.379 -bzip2: -- '1' -c_ares: -- '1' -c_compiler: -- vs2019 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- None -cuda_compiler_version_min: -- '10.2' -cxx_compiler: -- vs2019 -gflags: -- '2.2' -glog: -- '0.6' -google_cloud_cpp: -- 2.5.0 -libabseil: -- '20220623.0' -libcrc32c: -- '1.1' -libcurl: -- '7' -libgrpc: -- '1.49' -libprotobuf: -- '3.21' -lz4_c: -- 1.9.3 -numpy: -- '1.21' -- '1.23' -- '1.20' -- '1.20' -openssl: -- 1.1.1 -pin_run_as_build: - python: - min_pin: x.x - max_pin: x.x -python: -- 3.10.* *_cpython -- 3.11.* *_cpython -- 3.8.* *_cpython -- 3.9.* *_cpython -re2: -- 2022.06.01 -snappy: -- '1' -target_platform: -- win-64 -thrift_cpp: -- 0.16.0 -zip_keys: -- - python - - numpy -zlib: -- '1.2' -zstd: -- '1.5' diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 50ed65c7456c2..86d8e1c1b3d5f 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -267,19 +267,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda - conda-linux-x64-cpu-py3-openssl1.1.1: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_64_cuda_compiler_versionNoneopenssl1.1.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - - conda-linux-x64-cpu-py3-openssl3: + conda-linux-x64-cpu-py3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -291,19 +279,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-linux-x64-cuda-py3-openssl1.1.1: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_64_cuda_compiler_version10.2openssl1.1.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.conda - - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda - - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda - - conda-linux-x64-cuda-py3-openssl3: + conda-linux-x64-cuda-py3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -337,19 +313,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda - conda-linux-aarch64-cpu-py3-openssl1.1.1: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_aarch64_cuda_compiler_versionNoneopenssl1.1.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - - conda-linux-aarch64-cpu-py3-openssl3: + conda-linux-aarch64-cpu-py3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -363,19 +327,7 @@ tasks: ########################### Conda Linux (ppc64le) ########################### - conda-linux-ppc64le-cpu-py3-openssl1.1.1: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_ppc64le_cuda_compiler_versionNoneopenssl1.1.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - - conda-linux-ppc64le-cpu-py3-openssl3: + conda-linux-ppc64le-cpu-py3: ci: azure template: conda-recipes/azure.linux.yml params: @@ -409,19 +361,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda - conda-osx-x64-cpu-py3-openssl1.1.1: - ci: azure - template: conda-recipes/azure.osx.yml - params: - config: osx_64_openssl1.1.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - - conda-osx-x64-cpu-py3-openssl3: + conda-osx-x64-cpu-py3: ci: azure template: conda-recipes/azure.osx.yml params: @@ -455,19 +395,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r42(h[a-z0-9]+)_0.conda - conda-osx-arm64-cpu-py3-openssl1.1.1: - ci: azure - template: conda-recipes/azure.osx.yml - params: - config: osx_arm64_openssl1.1.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - - conda-osx-arm64-cpu-py3-openssl3: + conda-osx-arm64-cpu-py3: ci: azure template: conda-recipes/azure.osx.yml params: @@ -491,19 +419,7 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-win-x64-cpu-py3-openssl1.1.1: - ci: azure - template: conda-recipes/azure.win.yml - params: - config: win_64_cuda_compiler_versionNoneopenssl1.1.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - - conda-win-x64-cpu-py3-openssl3: + conda-win-x64-cpu-py3: ci: azure template: conda-recipes/azure.win.yml params: @@ -515,19 +431,7 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda - conda-win-x64-cuda-py3-openssl1.1.1: - ci: azure - template: conda-recipes/azure.win.yml - params: - config: win_64_cuda_compiler_versionNoneopenssl1.1.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda - - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda - - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.conda - - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda - - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda - - conda-win-x64-cuda-py3-openssl3: + conda-win-x64-cuda-py3: ci: azure template: conda-recipes/azure.win.yml params: From af6df5ff816d3927238fc8bcbf3ec364fabd5895 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 5 Dec 2022 20:40:04 +1100 Subject: [PATCH 14/25] fix C&P error; add comment about missing R 4.2 on windows --- dev/tasks/tasks.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 86d8e1c1b3d5f..9799a3709d82c 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -377,7 +377,7 @@ tasks: conda-osx-arm64-cpu-r41: ci: azure - template: conda-recipes/azure.linux.yml + template: conda-recipes/azure.osx.yml params: config: osx_arm64_openssl3 r_config: osx_arm64_r_base4.1 @@ -387,7 +387,7 @@ tasks: conda-osx-arm64-cpu-r42: ci: azure - template: conda-recipes/azure.linux.yml + template: conda-recipes/azure.osx.yml params: config: osx_arm64_openssl3 r_config: osx_arm64_r_base4.2 @@ -419,6 +419,8 @@ tasks: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + # conda-forge does not yet support R 4.2 on windows + conda-win-x64-cpu-py3: ci: azure template: conda-recipes/azure.win.yml From c17821960b142b37a9f841c7355d5d881a7d7696 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 5 Dec 2022 20:13:09 +1100 Subject: [PATCH 15/25] fix one more failure mode for instantiating gcsfs fixture this avoids blowing up CI time by 30+ minutes due to running into unnecessary timeouts, by skipping earlier (if testbench not found) --- python/pyarrow/tests/conftest.py | 7 ++++++- python/pyarrow/tests/test_fs.py | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index 83381c982a57e..56bce805773f2 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -184,11 +184,16 @@ def gcs_server(): args = [sys.executable, '-m', 'testbench', '--port', str(port)] proc = None try: + # check first if testbench module is available + import testbench + # start server proc = subprocess.Popen(args, env=env) # Make sure the server is alive. assert proc.poll() is None - except (AssertionError, OSError) as e: + except (ModuleNotFoundError, OSError) as e: pytest.skip(f"Command {args} failed to execute: {e}") + except AssertionError as e: + pytest.skip(f"Command {args} did not start server successfully: {e}") else: yield { 'connection': ('localhost', port), diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index 1af57ea9407d9..b6e3456f5d3f4 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -215,7 +215,10 @@ def gcsfs(request, gcs_server): anonymous=True, retry_time_limit=timedelta(seconds=45) ) - fs.create_dir(bucket) + try: + fs.create_dir(bucket) + except OSError as e: + pytest.skip(f"Could not create directory in {fs}: {e}") yield dict( fs=fs, From 4dd7a43f174d69d37c59cd9572a1c10d6675a7d9 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 6 Dec 2022 00:06:28 +1100 Subject: [PATCH 16/25] appease linter --- python/pyarrow/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index 56bce805773f2..38b8a4cf0e95b 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -185,7 +185,7 @@ def gcs_server(): proc = None try: # check first if testbench module is available - import testbench + import testbench # noqa:F401 # start server proc = subprocess.Popen(args, env=env) # Make sure the server is alive. From 85e742e08c3822db1df507cfc2c888d0eeed5eab Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 6 Dec 2022 09:30:00 +1100 Subject: [PATCH 17/25] replace assert with if for checking if server is alive --- python/pyarrow/tests/conftest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index 38b8a4cf0e95b..e6d87217eca16 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -189,11 +189,10 @@ def gcs_server(): # start server proc = subprocess.Popen(args, env=env) # Make sure the server is alive. - assert proc.poll() is None + if proc.poll() is not None: + pytest.skip(f"Command {args} did not start server successfully!") except (ModuleNotFoundError, OSError) as e: pytest.skip(f"Command {args} failed to execute: {e}") - except AssertionError as e: - pytest.skip(f"Command {args} did not start server successfully: {e}") else: yield { 'connection': ('localhost', port), From 643220b4af58b572604c06ba14a744acdd9881fd Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 6 Dec 2022 09:38:59 +1100 Subject: [PATCH 18/25] run test suite for all python versions on linux64 --- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 3d21004d531cc..1c3478582b5fe 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -383,8 +383,8 @@ outputs: {% set tests_to_skip = tests_to_skip + " or (test_misc and test_get_include)" %} # [unix] # ^^^^^^^ TESTS THAT SHOULDN'T HAVE TO BE SKIPPED ^^^^^^^ - # crossbow CI: enough to run for one python version - - pytest -v -rfEs -k "not ({{ tests_to_skip }})" # [(not ppc64le) and py==310] + # crossbow CI: reduce to one python version, except on (unemulated) linux, where it's fast enough + - pytest -v -rfEs -k "not ({{ tests_to_skip }})" # [linux64 or ((py==310) and not ppc64le)] # NOTE: test suite is skipped on ppc due to bugs in QEMU code that cause # CI to fail, even though the tests would run through on native hardware From 0e3ce74cd8327ad6a10773ae9ef2d87c07da8524 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 6 Dec 2022 11:46:50 +1100 Subject: [PATCH 19/25] better C++17 comment --- dev/tasks/conda-recipes/r-arrow/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/conda-recipes/r-arrow/build.sh b/dev/tasks/conda-recipes/r-arrow/build.sh index 5abe54e38f8b3..c02f989650cb2 100644 --- a/dev/tasks/conda-recipes/r-arrow/build.sh +++ b/dev/tasks/conda-recipes/r-arrow/build.sh @@ -3,7 +3,7 @@ set -ex export DISABLE_AUTOBREW=1 -# set C++17 due to abseil +# arrow uses C++17 export ARROW_R_CXXFLAGS="${ARROW_R_CXXFLAGS} -std=c++17" if [[ "${target_platform}" == osx-* ]]; then From b4147098fcdd229964bf3c97da129e2a5e4265e9 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 6 Dec 2022 13:12:19 +1100 Subject: [PATCH 20/25] ensure we don't try to run tests on osx-arm --- dev/tasks/conda-recipes/build_steps.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh index dfc468c0e4420..ea607763fc5a2 100755 --- a/dev/tasks/conda-recipes/build_steps.sh +++ b/dev/tasks/conda-recipes/build_steps.sh @@ -36,6 +36,10 @@ source run_conda_forge_build_setup # make the build number clobber make_build_number "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}" +if [[ "${HOST_PLATFORM}" != "${BUILD_PLATFORM}" ]] && [[ "${HOST_PLATFORM}" != linux-* ]] && [[ "${BUILD_WITH_CONDA_DEBUG:-0}" != 1 ]]; then + EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test" +fi + export CONDA_BLD_PATH="${output_dir}" conda mambabuild \ @@ -43,13 +47,15 @@ conda mambabuild \ "${FEEDSTOCK_ROOT}/parquet-cpp" \ -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" \ - --output-folder "${output_dir}" + --output-folder "${output_dir}" \ + ${EXTRA_CB_OPTIONS:-} if [ ! -z "${R_CONFIG:-}" ]; then conda mambabuild \ "${FEEDSTOCK_ROOT}/r-arrow" \ -m "${CI_SUPPORT}/r/${R_CONFIG}.yaml" \ - --output-folder "${output_dir}" + --output-folder "${output_dir}" \ + ${EXTRA_CB_OPTIONS:-} fi From df81110c040891b0b3b2350a31cf4a97a380f746 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 6 Dec 2022 17:19:46 +1100 Subject: [PATCH 21/25] one last missing EXTRA_CB_OPTIONS --- dev/tasks/conda-recipes/azure.osx.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index 404d6daee3bb1..bf0ac043213b3 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -71,6 +71,7 @@ jobs: if [ ! -z "${R_CONFIG}" ]; then conda build r-arrow \ -m ./.ci_support/r/${R_CONFIG}.yaml \ + ${EXTRA_CB_OPTIONS:-} \ --output-folder ./build_artifacts fi workingDirectory: arrow/dev/tasks/conda-recipes From d81cec9e9ed445f3b5a4520512b9e155ecdb7ba4 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 6 Dec 2022 17:22:01 +1100 Subject: [PATCH 22/25] update artefact extension in azure..yml --- dev/tasks/conda-recipes/azure.linux.yml | 4 ++-- dev/tasks/conda-recipes/azure.osx.yml | 4 ++-- dev/tasks/conda-recipes/azure.win.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index c05d284d25f73..b9a54647cc525 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -34,5 +34,5 @@ jobs: CI=azure arrow/dev/tasks/conda-recipes/run_docker_build.sh $(pwd)/build_artifacts displayName: Run docker build - {{ macros.azure_upload_releases("build_artifacts/*/*.tar.bz2") }} - {{ macros.azure_upload_anaconda("build_artifacts/*/*.tar.bz2") }} + {{ macros.azure_upload_releases("build_artifacts/*/*.conda") }} + {{ macros.azure_upload_anaconda("build_artifacts/*/*.conda") }} diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index bf0ac043213b3..a6440086dbede 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -81,5 +81,5 @@ jobs: sudo mv /usr/local/conda_mangled/* /usr/local/ displayName: Unmangle homebrew - {{ macros.azure_upload_releases("arrow/dev/tasks/conda-recipes/build_artifacts/osx-*/*.tar.bz2") }} - {{ macros.azure_upload_anaconda("arrow/dev/tasks/conda-recipes/build_artifacts/osx-*/*.tar.bz2") }} + {{ macros.azure_upload_releases("arrow/dev/tasks/conda-recipes/build_artifacts/osx-*/*.conda") }} + {{ macros.azure_upload_anaconda("arrow/dev/tasks/conda-recipes/build_artifacts/osx-*/*.conda") }} diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml index 6aac0eeda9066..95c5fc1005686 100755 --- a/dev/tasks/conda-recipes/azure.win.yml +++ b/dev/tasks/conda-recipes/azure.win.yml @@ -72,5 +72,5 @@ jobs: PYTHONUNBUFFERED: 1 condition: contains(variables['R_CONFIG'], 'win') - {{ macros.azure_upload_releases("D:\\bld\\win-64\\*.tar.bz2")|indent(2) }} - {{ macros.azure_upload_anaconda("D:\\bld\\win-64\\*.tar.bz2")|indent(2) }} + {{ macros.azure_upload_releases("D:\\bld\\win-64\\*.conda")|indent(2) }} + {{ macros.azure_upload_anaconda("D:\\bld\\win-64\\*.conda")|indent(2) }} From 25333962c5ae23d2e9e7a5c5bbae1dce2b5bb08c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 6 Dec 2022 23:38:53 +1100 Subject: [PATCH 23/25] use the same macOS image as conda-forge --- dev/tasks/conda-recipes/azure.osx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index a6440086dbede..8b1db992ec77c 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -3,7 +3,7 @@ jobs: - job: osx pool: - vmImage: macOS-latest + vmImage: macOS-11 timeoutInMinutes: 360 variables: CONFIG: {{ config }} From c5fcd6c855acd48d4fc10d174d5078bc9c87a3a6 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 7 Dec 2022 08:37:00 +1100 Subject: [PATCH 24/25] add comment as requested --- dev/tasks/conda-recipes/r-arrow/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/conda-recipes/r-arrow/build.sh b/dev/tasks/conda-recipes/r-arrow/build.sh index c02f989650cb2..8479d9db5c415 100644 --- a/dev/tasks/conda-recipes/r-arrow/build.sh +++ b/dev/tasks/conda-recipes/r-arrow/build.sh @@ -11,4 +11,5 @@ if [[ "${target_platform}" == osx-* ]]; then export ARROW_R_CXXFLAGS="${ARROW_R_CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" fi +# ${R_ARGS} necessary to support cross-compilation ${R} CMD INSTALL --build r/. ${R_ARGS} From dcd4fcc9846ecc67313113831466a91383a24203 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 7 Dec 2022 10:19:33 +1100 Subject: [PATCH 25/25] reflect latest grpc migration in .ci_support --- .../.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml | 2 +- .../.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml | 2 +- .../linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml | 2 +- .../linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml | 2 +- dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml | 2 +- dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml | 2 +- .../.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml | 2 +- .../.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml index 5153ba8d32c9d..31c61b17fc021 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml @@ -35,7 +35,7 @@ google_cloud_cpp: libabseil: - '20220623.0' libgrpc: -- '1.49' +- '1.51' libprotobuf: - '3.21' lz4_c: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml index e22596823b30a..e0629f862784d 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml @@ -35,7 +35,7 @@ google_cloud_cpp: libabseil: - '20220623.0' libgrpc: -- '1.49' +- '1.51' libprotobuf: - '3.21' lz4_c: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml index a07a0762efb33..793c1afa7ab9b 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml @@ -35,7 +35,7 @@ google_cloud_cpp: libabseil: - '20220623.0' libgrpc: -- '1.49' +- '1.51' libprotobuf: - '3.21' lz4_c: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml index 25c42f8ccf06b..6c81ae03b6d4d 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml @@ -31,7 +31,7 @@ google_cloud_cpp: libabseil: - '20220623.0' libgrpc: -- '1.49' +- '1.51' libprotobuf: - '3.21' lz4_c: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml index 1408d2f0e6d11..bff4181d54925 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml @@ -29,7 +29,7 @@ google_cloud_cpp: libabseil: - '20220623.0' libgrpc: -- '1.49' +- '1.51' libprotobuf: - '3.21' lz4_c: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml index 56f28d01bcf98..2e9a5682b9564 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml @@ -29,7 +29,7 @@ google_cloud_cpp: libabseil: - '20220623.0' libgrpc: -- '1.49' +- '1.51' libprotobuf: - '3.21' lz4_c: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml index 66f031b56f035..7173299488eea 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml @@ -31,7 +31,7 @@ libcrc32c: libcurl: - '7' libgrpc: -- '1.49' +- '1.51' libprotobuf: - '3.21' lz4_c: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml index b18fc222a8004..49fc4defc1bbb 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml @@ -31,7 +31,7 @@ libcrc32c: libcurl: - '7' libgrpc: -- '1.49' +- '1.51' libprotobuf: - '3.21' lz4_c: