From 1d54a4f3e48665253ccb1d9f57b2a1c54cfc3aaf Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Sun, 25 Jan 2026 14:19:57 -0600 Subject: [PATCH 01/36] Starting changes --- cpp/src/parquet/meson.build | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index e6ff43a0bae..fe10f650aed 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -99,8 +99,15 @@ if openssl_dep.found() parquet_deps += openssl_dep parquet_srcs += files( + 'encryption/aes_encryption.cc', 'encryption/crypto_factory.cc', - 'encryption/encryption_internal.cc', + 'encryption/encoding_properties', + 'encryption/encryption_utils.cc', + 'encryption/external_dbpa_encryption.cc', + 'encryption/external/dbpa_library_wrapper', + 'encryption/external/dbpa_enum_utils', + 'encryption/external/dbpa_executor', + 'encryption/external/loadable_encryptor_utils', 'encryption/file_key_unwrapper.cc', 'encryption/file_key_wrapper.cc', 'encryption/file_system_key_material_store.cc', @@ -113,7 +120,7 @@ if openssl_dep.found() 'encryption/openssl_internal.cc', ) else - parquet_srcs += files('encryption/encryption_internal_nossl.cc') + parquet_srcs += files('encryption/aes_encryption_nossl.cc') endif parquet_lib = library( From 601849e32d3b877e91e618c9db7f5bd2dd9af1fc Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 18:20:49 -0600 Subject: [PATCH 02/36] First attempt at adding new external encryption to meson.build --- cpp/src/parquet/meson.build | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index fe10f650aed..c66fcc7034e 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -223,6 +223,7 @@ parquet_tests = { 'writer-test': { 'sources': files( 'column_writer_test.cc', + 'encryption/external/test_utils.cc', 'file_serialize_test.cc', 'stream_writer_test.cc', ), @@ -255,7 +256,12 @@ if needs_parquet_encryption parquet_tests += { 'encryption-test': { 'sources': files( - 'encryption/encryption_internal_test.cc', + 'encryption/aes_encryption_test.cc', + 'encryption/crypto_factory_test.cc', + 'encryption/encoding_properties_test.cc', + 'encryption/external_dbpa_encryption_test.cc', + 'encryption/external/test_utils.cc', + 'encryption/per_column_encryption_test.cc', 'encryption/properties_test.cc', 'encryption/read_configurations_test.cc', 'encryption/test_encryption_util.cc', From cdd51ccfbd793d056212b39da4e47ac8d215ed04 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 18:26:48 -0600 Subject: [PATCH 03/36] Fixing typo --- cpp/src/parquet/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index c66fcc7034e..f92b20bd14f 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -101,7 +101,7 @@ if openssl_dep.found() parquet_srcs += files( 'encryption/aes_encryption.cc', 'encryption/crypto_factory.cc', - 'encryption/encoding_properties', + 'encryption/encoding_properties.cc', 'encryption/encryption_utils.cc', 'encryption/external_dbpa_encryption.cc', 'encryption/external/dbpa_library_wrapper', From f65cc16eef8c300d1e64a4fce6f892072943e9d8 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 18:32:19 -0600 Subject: [PATCH 04/36] Fixing typo --- cpp/src/parquet/meson.build | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index f92b20bd14f..1629dd305fa 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -104,10 +104,10 @@ if openssl_dep.found() 'encryption/encoding_properties.cc', 'encryption/encryption_utils.cc', 'encryption/external_dbpa_encryption.cc', - 'encryption/external/dbpa_library_wrapper', - 'encryption/external/dbpa_enum_utils', - 'encryption/external/dbpa_executor', - 'encryption/external/loadable_encryptor_utils', + 'encryption/external/dbpa_library_wrapper.cc', + 'encryption/external/dbpa_enum_utils.cc', + 'encryption/external/dbpa_executor.cc', + 'encryption/external/loadable_encryptor_utils.cc', 'encryption/file_key_unwrapper.cc', 'encryption/file_key_wrapper.cc', 'encryption/file_system_key_material_store.cc', From 7b88e4b865cf31d04a059fd4aecbee72ca351155 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 19:08:44 -0600 Subject: [PATCH 05/36] Fixing typo --- cpp/src/parquet/meson.build | 40 +++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 1629dd305fa..35e5ac7aba8 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -123,6 +123,46 @@ else parquet_srcs += files('encryption/aes_encryption_nossl.cc') endif +if needs_parquet_encryption or get_option('parquet_require_encryption').auto() + dbps_source_dir = meson.project_source_root() / 'subprojects' / 'dbps_agent' + dbps_interface_dep = declare_dependency( + include_directories: include_directories( + dbps_source_dir / 'src' / 'common' + ) + ) + + parquet_build_dbps_libs = get_option('parquet_build_dbps_libs') + + if parquet_build_dbps_libs.allowed() + cmake = import('cmake') + dbps_opts = cmake.subproject_options() + + build_type = get_option('build_type') + if build_type == 'plain' + build_type = 'Release' + elif build_type == 'debug' + build_type = 'Debug' + endif + + build_output_dir = meson.current_build_dir() + + dbps_opts.add_cmake_defines( + { + 'CMAKE_BUILD_TYPE': build_type, + 'BUILD_SHARED_LIBS': 'ON', + 'BUILD_TESTING': 'OFF', + 'CMAKE_LIBRARY_OUTPUT_DIRECTORY': build_output_dir, + 'CMAKE_RUNTIME_OUTPUT_DIRECTORY': build_output_dir, + 'CMAKE_ARCHIVE_OUTPUT_DIRECTORY': build_output_dir, + } + ) + + dbps_proj = cmake.subproject('dbps_agent', options: dbps_opts) + endif + + parquet_deps += dbps_interface_dep +endif + parquet_lib = library( 'arrow-parquet', sources: parquet_srcs, From c6207eda98833a6989619e6cc40c962697f139bf Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 19:09:58 -0600 Subject: [PATCH 06/36] Adding dbps external project --- cpp/meson.options | 6 ++++++ cpp/subprojects/dbps_agent.wrap | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 cpp/subprojects/dbps_agent.wrap diff --git a/cpp/meson.options b/cpp/meson.options index 668f440ee72..af7dff7bde9 100644 --- a/cpp/meson.options +++ b/cpp/meson.options @@ -90,6 +90,12 @@ option( description: 'Arbitrary string that identifies the kind of package (for informational purposes)', ) option('parquet', type: 'feature', description: 'Build the Parquet libraries') +option( + 'parquet_build_dbps_libs', + type: 'feature', + value: 'enabled', + description: 'Build DBPS external libraries' +) option( 'parquet_build_executables', type: 'feature', diff --git a/cpp/subprojects/dbps_agent.wrap b/cpp/subprojects/dbps_agent.wrap new file mode 100644 index 00000000000..232444cbf67 --- /dev/null +++ b/cpp/subprojects/dbps_agent.wrap @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-git] +url = https://github.com/protegrity/DataBatchProtectionService.git +# TODO: Change to a specific tag/commit when we have one. +# https://github.com/protegrity/arrow/issues/179 +revision = 6206fb0e27556a0df9160364caa3819e4af3fe0f +depth = 1 From 3d8bb08c75dc2fafdc19827a56320fd24e78f820 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 19:18:13 -0600 Subject: [PATCH 07/36] Fix path --- cpp/src/parquet/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 35e5ac7aba8..62e278f898c 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -127,7 +127,7 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() dbps_source_dir = meson.project_source_root() / 'subprojects' / 'dbps_agent' dbps_interface_dep = declare_dependency( include_directories: include_directories( - dbps_source_dir / 'src' / 'common' + '../../subprojects/dbps_agent/src/common' ) ) From 91f196da7b8a3c24b5ddd76aeb618966ba37eea0 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 19:44:34 -0600 Subject: [PATCH 08/36] Restructuring subproject --- cpp/src/parquet/meson.build | 7 +++--- cpp/subprojects/dbps_agent.wrap | 3 +++ .../packagefiles/dbps_agent/meson.build | 24 +++++++++++++++++++ 3 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 cpp/subprojects/packagefiles/dbps_agent/meson.build diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 62e278f898c..89f5afaa4d2 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -125,12 +125,11 @@ endif if needs_parquet_encryption or get_option('parquet_require_encryption').auto() dbps_source_dir = meson.project_source_root() / 'subprojects' / 'dbps_agent' - dbps_interface_dep = declare_dependency( - include_directories: include_directories( - '../../subprojects/dbps_agent/src/common' - ) + dbps_interface_dep = dependency( + 'dbps_interface', fallback: ['dbps_agent', 'dbps_interface_dep'] ) + parquet_build_dbps_libs = get_option('parquet_build_dbps_libs') if parquet_build_dbps_libs.allowed() diff --git a/cpp/subprojects/dbps_agent.wrap b/cpp/subprojects/dbps_agent.wrap index 232444cbf67..59d2dac4575 100644 --- a/cpp/subprojects/dbps_agent.wrap +++ b/cpp/subprojects/dbps_agent.wrap @@ -21,3 +21,6 @@ url = https://github.com/protegrity/DataBatchProtectionService.git # https://github.com/protegrity/arrow/issues/179 revision = 6206fb0e27556a0df9160364caa3819e4af3fe0f depth = 1 + +[provide] +dbps_interface = dbps_interface_dep diff --git a/cpp/subprojects/packagefiles/dbps_agent/meson.build b/cpp/subprojects/packagefiles/dbps_agent/meson.build new file mode 100644 index 00000000000..9e3d2ff43e9 --- /dev/null +++ b/cpp/subprojects/packagefiles/dbps_agent/meson.build @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +project('dbps_agent', 'cpp') + +dbps_interface_dep = declare_dependency( + include_directories: include_directories('src/common') +) + +meson.override_dependency('dbps_interface', dbps_interface_dep) From efc52ef48de098c0a578261cafa06f7fd879f5d1 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 20:37:35 -0600 Subject: [PATCH 09/36] Different dependency --- cpp/src/parquet/meson.build | 8 +++++--- cpp/subprojects/dbps_agent.wrap | 2 -- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 89f5afaa4d2..99119b5a8d7 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -124,9 +124,11 @@ else endif if needs_parquet_encryption or get_option('parquet_require_encryption').auto() - dbps_source_dir = meson.project_source_root() / 'subprojects' / 'dbps_agent' - dbps_interface_dep = dependency( - 'dbps_interface', fallback: ['dbps_agent', 'dbps_interface_dep'] + subproject('dbps_agent', required: false) + dbps_interface_dep = declare_dependency( + include_directories: include_directories( + '../../subprojects/dbps_agent/src/common' + ) ) diff --git a/cpp/subprojects/dbps_agent.wrap b/cpp/subprojects/dbps_agent.wrap index 59d2dac4575..674553e92e9 100644 --- a/cpp/subprojects/dbps_agent.wrap +++ b/cpp/subprojects/dbps_agent.wrap @@ -22,5 +22,3 @@ url = https://github.com/protegrity/DataBatchProtectionService.git revision = 6206fb0e27556a0df9160364caa3819e4af3fe0f depth = 1 -[provide] -dbps_interface = dbps_interface_dep From 0e83c18115b9b22d629fad016327064dd6438344 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 20:48:59 -0600 Subject: [PATCH 10/36] Another attempt --- cpp/src/parquet/meson.build | 8 +++----- cpp/subprojects/dbps_agent.wrap | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 99119b5a8d7..67f7277a8b7 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -125,10 +125,8 @@ endif if needs_parquet_encryption or get_option('parquet_require_encryption').auto() subproject('dbps_agent', required: false) - dbps_interface_dep = declare_dependency( - include_directories: include_directories( - '../../subprojects/dbps_agent/src/common' - ) + dbps_interface_dep = dependency( + 'dbps_interface', fallback: ['dbps_agent', 'dbps_interface_dep'] ) @@ -138,7 +136,7 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() cmake = import('cmake') dbps_opts = cmake.subproject_options() - build_type = get_option('build_type') + build_type = get_option('buildtype') if build_type == 'plain' build_type = 'Release' elif build_type == 'debug' diff --git a/cpp/subprojects/dbps_agent.wrap b/cpp/subprojects/dbps_agent.wrap index 674553e92e9..d934695eda8 100644 --- a/cpp/subprojects/dbps_agent.wrap +++ b/cpp/subprojects/dbps_agent.wrap @@ -15,10 +15,19 @@ # specific language governing permissions and limitations # under the License. -[wrap-git] -url = https://github.com/protegrity/DataBatchProtectionService.git +#[wrap-git] +#url = https://github.com/protegrity/DataBatchProtectionService.git # TODO: Change to a specific tag/commit when we have one. # https://github.com/protegrity/arrow/issues/179 -revision = 6206fb0e27556a0df9160364caa3819e4af3fe0f -depth = 1 +#revision = 6206fb0e27556a0df9160364caa3819e4af3fe0f +#depth = 1 +[wrap-file] +directory = dbps_agent-6206fb0e27556a0df9160364caa3819e4af3fe0f +source_url = https://github.com/protegrity/DataBatchProtectionService/archive/6206fb0e27556a0df9160364caa3819e4af3fe0f.tar.gz +source_filename = dbps_agent-6206fb0e27556a0df9160364caa3819e4af3fe0f.tar.gz +source_hash = 9c95a1fec0c9851867a776c3241d3feb59b07bd7a50e653d6214e07a8ad62419 +patch_directory = dbps_agent + +[provide] +dbps_interface = dbps_interface_dep From 701866e2b9594f7f455ecd73318255bbe7b4f49a Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 21:01:06 -0600 Subject: [PATCH 11/36] Correcting directory name --- cpp/subprojects/dbps_agent.wrap | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/cpp/subprojects/dbps_agent.wrap b/cpp/subprojects/dbps_agent.wrap index d934695eda8..81694918385 100644 --- a/cpp/subprojects/dbps_agent.wrap +++ b/cpp/subprojects/dbps_agent.wrap @@ -15,15 +15,8 @@ # specific language governing permissions and limitations # under the License. -#[wrap-git] -#url = https://github.com/protegrity/DataBatchProtectionService.git -# TODO: Change to a specific tag/commit when we have one. -# https://github.com/protegrity/arrow/issues/179 -#revision = 6206fb0e27556a0df9160364caa3819e4af3fe0f -#depth = 1 - [wrap-file] -directory = dbps_agent-6206fb0e27556a0df9160364caa3819e4af3fe0f +directory = DataBatchProtectionService-6206fb0e27556a0df9160364caa3819e4af3fe0f source_url = https://github.com/protegrity/DataBatchProtectionService/archive/6206fb0e27556a0df9160364caa3819e4af3fe0f.tar.gz source_filename = dbps_agent-6206fb0e27556a0df9160364caa3819e4af3fe0f.tar.gz source_hash = 9c95a1fec0c9851867a776c3241d3feb59b07bd7a50e653d6214e07a8ad62419 From 0c409b47f84bb1a5c93a52b96bcb2cf865f102c4 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 21:14:30 -0600 Subject: [PATCH 12/36] Another attempt --- cpp/src/parquet/meson.build | 8 +++++--- cpp/subprojects/dbps_agent.wrap | 3 --- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 67f7277a8b7..91e679bd18a 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -124,9 +124,11 @@ else endif if needs_parquet_encryption or get_option('parquet_require_encryption').auto() - subproject('dbps_agent', required: false) - dbps_interface_dep = dependency( - 'dbps_interface', fallback: ['dbps_agent', 'dbps_interface_dep'] + dbps_source_dir = meson.project_source_root() / 'subprojects' / 'DataBatchProtectionService-6206fb0e27556a0df9160364caa3819e4af3fe0f' + dbps_interface_dep = declare_dependency( + include_directories: include_directories( + dbps_source_dir / 'src' / 'common' + ) ) diff --git a/cpp/subprojects/dbps_agent.wrap b/cpp/subprojects/dbps_agent.wrap index 81694918385..ac8c37bf1cb 100644 --- a/cpp/subprojects/dbps_agent.wrap +++ b/cpp/subprojects/dbps_agent.wrap @@ -20,7 +20,4 @@ directory = DataBatchProtectionService-6206fb0e27556a0df9160364caa3819e4af3fe0f source_url = https://github.com/protegrity/DataBatchProtectionService/archive/6206fb0e27556a0df9160364caa3819e4af3fe0f.tar.gz source_filename = dbps_agent-6206fb0e27556a0df9160364caa3819e4af3fe0f.tar.gz source_hash = 9c95a1fec0c9851867a776c3241d3feb59b07bd7a50e653d6214e07a8ad62419 -patch_directory = dbps_agent -[provide] -dbps_interface = dbps_interface_dep From c88ad326c9691069d6ad18b39918805c09e41056 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 21:26:16 -0600 Subject: [PATCH 13/36] Removing absolute paths --- cpp/src/parquet/meson.build | 63 ++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 91e679bd18a..3a16c32cec5 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -124,41 +124,38 @@ else endif if needs_parquet_encryption or get_option('parquet_require_encryption').auto() - dbps_source_dir = meson.project_source_root() / 'subprojects' / 'DataBatchProtectionService-6206fb0e27556a0df9160364caa3819e4af3fe0f' - dbps_interface_dep = declare_dependency( - include_directories: include_directories( - dbps_source_dir / 'src' / 'common' - ) - ) - - - parquet_build_dbps_libs = get_option('parquet_build_dbps_libs') - - if parquet_build_dbps_libs.allowed() - cmake = import('cmake') - dbps_opts = cmake.subproject_options() - - build_type = get_option('buildtype') - if build_type == 'plain' - build_type = 'Release' - elif build_type == 'debug' - build_type = 'Debug' - endif - - build_output_dir = meson.current_build_dir() + cmake = import('cmake') + dbps_opts = cmake.subproject_options() - dbps_opts.add_cmake_defines( - { - 'CMAKE_BUILD_TYPE': build_type, - 'BUILD_SHARED_LIBS': 'ON', - 'BUILD_TESTING': 'OFF', - 'CMAKE_LIBRARY_OUTPUT_DIRECTORY': build_output_dir, - 'CMAKE_RUNTIME_OUTPUT_DIRECTORY': build_output_dir, - 'CMAKE_ARCHIVE_OUTPUT_DIRECTORY': build_output_dir, - } - ) + build_type = get_option('buildtype') + if build_type == 'plain' + build_type = 'Release' + elif build_type == 'debug' + build_type = 'Debug' + endif - dbps_proj = cmake.subproject('dbps_agent', options: dbps_opts) + build_output_dir = meson.current_build_dir() + + dbps_opts.add_cmake_defines({ + 'CMAKE_BUILD_TYPE': build_type, + 'BUILD_SHARED_LIBS': 'ON', + 'BUILD_TESTING': 'OFF', + 'CMAKE_LIBRARY_OUTPUT_DIRECTORY': build_output_dir, + 'CMAKE_RUNTIME_OUTPUT_DIRECTORY': build_output_dir, + 'CMAKE_ARCHIVE_OUTPUT_DIRECTORY': build_output_dir, + }) + + dbps_proj = cmake.subproject('dbps_agent', options: dbps_opts) + dbps_interface_dep = dbps_proj.dependency('dbps_interface', required: false) + + if not dbps_interface_dep.found() + #dbps_source_dir = dbps_proj.subproject_dir() + #dbps_interface_dep = declare_dependency( + # include_directories: include_directories( + # dbps_source_dir / 'src' / 'common' + # ) + #) + error('No idea') endif parquet_deps += dbps_interface_dep From 24e247a314173d48a8352d10eeb5f51d16f2a020 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 21:33:55 -0600 Subject: [PATCH 14/36] Removing absolute paths --- cpp/src/parquet/meson.build | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 3a16c32cec5..842d3d3f612 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -146,16 +146,10 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() }) dbps_proj = cmake.subproject('dbps_agent', options: dbps_opts) - dbps_interface_dep = dbps_proj.dependency('dbps_interface', required: false) + dbps_interface_dep = dbps_proj.dependency('dbps_interface') if not dbps_interface_dep.found() - #dbps_source_dir = dbps_proj.subproject_dir() - #dbps_interface_dep = declare_dependency( - # include_directories: include_directories( - # dbps_source_dir / 'src' / 'common' - # ) - #) - error('No idea') + error('Failed to build DBPS project') endif parquet_deps += dbps_interface_dep From f8979f5523821828c0df3df1fe9912029f90e72b Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 21:41:57 -0600 Subject: [PATCH 15/36] Removing absolute paths --- cpp/src/parquet/meson.build | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 842d3d3f612..c488050fb57 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -146,7 +146,11 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() }) dbps_proj = cmake.subproject('dbps_agent', options: dbps_opts) - dbps_interface_dep = dbps_proj.dependency('dbps_interface') + dbps_interface_dep = declare_dependency( + include_directories: include_directories( + '../../subprojects/DataBatchProtectionService-6206fb0e27556a0df9160364caa3819e4af3fe0f/src' + ) + ) if not dbps_interface_dep.found() error('Failed to build DBPS project') From dad2c6268c9880250504269cec65680f2fd83614 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 22:06:57 -0600 Subject: [PATCH 16/36] Removing absolute paths --- cpp/src/parquet/meson.build | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index c488050fb57..a4a9b14f139 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -148,14 +148,10 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() dbps_proj = cmake.subproject('dbps_agent', options: dbps_opts) dbps_interface_dep = declare_dependency( include_directories: include_directories( - '../../subprojects/DataBatchProtectionService-6206fb0e27556a0df9160364caa3819e4af3fe0f/src' + '../../subprojects/DataBatchProtectionService-6206fb0e27556a0df9160364caa3819e4af3fe0f/src/common' ) ) - if not dbps_interface_dep.found() - error('Failed to build DBPS project') - endif - parquet_deps += dbps_interface_dep endif From 5602a8fe6a92a872121b5184e712b13f88848ee8 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 22:35:42 -0600 Subject: [PATCH 17/36] Add missing dependencies to tcb and magic enum --- cpp/src/parquet/meson.build | 7 +++++- cpp/subprojects/magic-enum.wrap | 24 +++++++++++++++++++ .../packagefiles/magic-enum/meson.build | 22 +++++++++++++++++ .../packagefiles/tcb-span/meson.build | 22 +++++++++++++++++ cpp/subprojects/tcb-span.wrap | 24 +++++++++++++++++++ 5 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 cpp/subprojects/magic-enum.wrap create mode 100644 cpp/subprojects/packagefiles/magic-enum/meson.build create mode 100644 cpp/subprojects/packagefiles/tcb-span/meson.build create mode 100644 cpp/subprojects/tcb-span.wrap diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index a4a9b14f139..5016f39d354 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -124,6 +124,11 @@ else endif if needs_parquet_encryption or get_option('parquet_require_encryption').auto() + tcb_span_dep = dependency('tcb_span', fallback: ['tcb-span', 'tcb_span_dep']) + magic_enum_dep = dependency( + 'magic_enum_header_only', fallback: ['magic-enum', 'magic_enum_dep'] + ) + cmake = import('cmake') dbps_opts = cmake.subproject_options() @@ -152,7 +157,7 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() ) ) - parquet_deps += dbps_interface_dep + parquet_deps += [dbps_interface_dep, tcb_span_dep, magic_enum_dep] endif parquet_lib = library( diff --git a/cpp/subprojects/magic-enum.wrap b/cpp/subprojects/magic-enum.wrap new file mode 100644 index 00000000000..3f7a17d9327 --- /dev/null +++ b/cpp/subprojects/magic-enum.wrap @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-git] +url = https://github.com/Neargye/magic_enum.git +revision = v0.9.7 +depth = 1 + +[provide] +magic_enum_header_only = magic_enum_dep \ No newline at end of file diff --git a/cpp/subprojects/packagefiles/magic-enum/meson.build b/cpp/subprojects/packagefiles/magic-enum/meson.build new file mode 100644 index 00000000000..3b1a57bca77 --- /dev/null +++ b/cpp/subprojects/packagefiles/magic-enum/meson.build @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +project('magic-enum', 'cpp') + +magic_enum_dep = declare_dependency(include_directories: include_directories('include')) + +meson.override_dependency('magic_enum_header_only', magic_enum_dep) diff --git a/cpp/subprojects/packagefiles/tcb-span/meson.build b/cpp/subprojects/packagefiles/tcb-span/meson.build new file mode 100644 index 00000000000..0487df497ed --- /dev/null +++ b/cpp/subprojects/packagefiles/tcb-span/meson.build @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +project('tcb-span', 'cpp') + +tcb_span_dep = declare_dependency(include_directories: include_directories('include')) + +meson.override_dependency('tcb_span', tcb_span_dep) diff --git a/cpp/subprojects/tcb-span.wrap b/cpp/subprojects/tcb-span.wrap new file mode 100644 index 00000000000..4d1f4a7390b --- /dev/null +++ b/cpp/subprojects/tcb-span.wrap @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-git] +url = https://github.com/tcbrindle/span.git +revision = master +depth = 1 + +[provide] +tcb_span = tcb_span_dep From b9f6899438a907995ebdc55d9a05347a7c71e50c Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 22:50:55 -0600 Subject: [PATCH 18/36] Changing dep download --- cpp/subprojects/magic-enum.wrap | 12 +++++++----- cpp/subprojects/tcb-span.wrap | 10 ++++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/cpp/subprojects/magic-enum.wrap b/cpp/subprojects/magic-enum.wrap index 3f7a17d9327..5e4b7c7da10 100644 --- a/cpp/subprojects/magic-enum.wrap +++ b/cpp/subprojects/magic-enum.wrap @@ -15,10 +15,12 @@ # specific language governing permissions and limitations # under the License. -[wrap-git] -url = https://github.com/Neargye/magic_enum.git -revision = v0.9.7 -depth = 1 +[wrap-file] +directory = magic_enum-0.9.7 +source_url = https://github.com/Neargye/magic_enum/archive/refs/tags/v0.9.7.tar.gz +source_filename = magic_enum-0.9.7.tar.gz +source_hash = b403d3dad4ef542fdc3024fa37d3a6cedb4ad33c72e31b6d9bab89dcaf69edf7 +patch_directory = magic-enum [provide] -magic_enum_header_only = magic_enum_dep \ No newline at end of file +magic_enum_header_only = magic_enum_dep diff --git a/cpp/subprojects/tcb-span.wrap b/cpp/subprojects/tcb-span.wrap index 4d1f4a7390b..fbe7e9a0e52 100644 --- a/cpp/subprojects/tcb-span.wrap +++ b/cpp/subprojects/tcb-span.wrap @@ -15,10 +15,12 @@ # specific language governing permissions and limitations # under the License. -[wrap-git] -url = https://github.com/tcbrindle/span.git -revision = master -depth = 1 +[wrap-file] +directory = span-master +source_url = https://github.com/tcbrindle/span/archive/refs/heads/master.tar.gz +source_filename = span-master.tar.gz +source_hash = e57658bd9c5984cebd8bd0758f242cbf4fa7ed1bd5251a883f68420c3b17f818 +patch_directory = tcb-span [provide] tcb_span = tcb_span_dep From 6858b577e6b8e004ea2cc81cae17fb99a383d8a8 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Mon, 26 Jan 2026 23:57:47 -0600 Subject: [PATCH 19/36] Linter and test dependencies --- cpp/meson.options | 2 +- cpp/src/parquet/meson.build | 34 +++++++++++-------- .../packagefiles/dbps_agent/meson.build | 2 +- .../packagefiles/magic-enum/meson.build | 4 ++- .../packagefiles/tcb-span/meson.build | 4 ++- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/cpp/meson.options b/cpp/meson.options index af7dff7bde9..bdd95d63006 100644 --- a/cpp/meson.options +++ b/cpp/meson.options @@ -94,7 +94,7 @@ option( 'parquet_build_dbps_libs', type: 'feature', value: 'enabled', - description: 'Build DBPS external libraries' + description: 'Build DBPS external libraries', ) option( 'parquet_build_executables', diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 5016f39d354..3dd88f42d57 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -103,11 +103,11 @@ if openssl_dep.found() 'encryption/crypto_factory.cc', 'encryption/encoding_properties.cc', 'encryption/encryption_utils.cc', - 'encryption/external_dbpa_encryption.cc', - 'encryption/external/dbpa_library_wrapper.cc', 'encryption/external/dbpa_enum_utils.cc', 'encryption/external/dbpa_executor.cc', + 'encryption/external/dbpa_library_wrapper.cc', 'encryption/external/loadable_encryptor_utils.cc', + 'encryption/external_dbpa_encryption.cc', 'encryption/file_key_unwrapper.cc', 'encryption/file_key_wrapper.cc', 'encryption/file_system_key_material_store.cc', @@ -126,7 +126,8 @@ endif if needs_parquet_encryption or get_option('parquet_require_encryption').auto() tcb_span_dep = dependency('tcb_span', fallback: ['tcb-span', 'tcb_span_dep']) magic_enum_dep = dependency( - 'magic_enum_header_only', fallback: ['magic-enum', 'magic_enum_dep'] + 'magic_enum_header_only', + fallback: ['magic-enum', 'magic_enum_dep'], ) cmake = import('cmake') @@ -141,19 +142,21 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() build_output_dir = meson.current_build_dir() - dbps_opts.add_cmake_defines({ - 'CMAKE_BUILD_TYPE': build_type, - 'BUILD_SHARED_LIBS': 'ON', - 'BUILD_TESTING': 'OFF', - 'CMAKE_LIBRARY_OUTPUT_DIRECTORY': build_output_dir, - 'CMAKE_RUNTIME_OUTPUT_DIRECTORY': build_output_dir, - 'CMAKE_ARCHIVE_OUTPUT_DIRECTORY': build_output_dir, - }) + dbps_opts.add_cmake_defines( + { + 'CMAKE_BUILD_TYPE': build_type, + 'BUILD_SHARED_LIBS': 'ON', + 'BUILD_TESTING': 'OFF', + 'CMAKE_LIBRARY_OUTPUT_DIRECTORY': build_output_dir, + 'CMAKE_RUNTIME_OUTPUT_DIRECTORY': build_output_dir, + 'CMAKE_ARCHIVE_OUTPUT_DIRECTORY': build_output_dir, + } + ) dbps_proj = cmake.subproject('dbps_agent', options: dbps_opts) dbps_interface_dep = declare_dependency( include_directories: include_directories( - '../../subprojects/DataBatchProtectionService-6206fb0e27556a0df9160364caa3819e4af3fe0f/src/common' + '../../subprojects/DataBatchProtectionService-6206fb0e27556a0df9160364caa3819e4af3fe0f/src/common', ) ) @@ -167,7 +170,10 @@ parquet_lib = library( gnu_symbol_visibility: 'inlineshidden', ) -parquet_dep = declare_dependency(link_with: parquet_lib) +parquet_dep = declare_dependency( + link_with: parquet_lib, + dependencies: parquet_deps +) subdir('api') subdir('arrow') @@ -296,8 +302,8 @@ if needs_parquet_encryption 'encryption/aes_encryption_test.cc', 'encryption/crypto_factory_test.cc', 'encryption/encoding_properties_test.cc', - 'encryption/external_dbpa_encryption_test.cc', 'encryption/external/test_utils.cc', + 'encryption/external_dbpa_encryption_test.cc', 'encryption/per_column_encryption_test.cc', 'encryption/properties_test.cc', 'encryption/read_configurations_test.cc', diff --git a/cpp/subprojects/packagefiles/dbps_agent/meson.build b/cpp/subprojects/packagefiles/dbps_agent/meson.build index 9e3d2ff43e9..527f11cac62 100644 --- a/cpp/subprojects/packagefiles/dbps_agent/meson.build +++ b/cpp/subprojects/packagefiles/dbps_agent/meson.build @@ -18,7 +18,7 @@ project('dbps_agent', 'cpp') dbps_interface_dep = declare_dependency( - include_directories: include_directories('src/common') + include_directories: include_directories('src/common'), ) meson.override_dependency('dbps_interface', dbps_interface_dep) diff --git a/cpp/subprojects/packagefiles/magic-enum/meson.build b/cpp/subprojects/packagefiles/magic-enum/meson.build index 3b1a57bca77..cdf68711c5b 100644 --- a/cpp/subprojects/packagefiles/magic-enum/meson.build +++ b/cpp/subprojects/packagefiles/magic-enum/meson.build @@ -17,6 +17,8 @@ project('magic-enum', 'cpp') -magic_enum_dep = declare_dependency(include_directories: include_directories('include')) +magic_enum_dep = declare_dependency( + include_directories: include_directories('include' +) meson.override_dependency('magic_enum_header_only', magic_enum_dep) diff --git a/cpp/subprojects/packagefiles/tcb-span/meson.build b/cpp/subprojects/packagefiles/tcb-span/meson.build index 0487df497ed..6e9d99e0cab 100644 --- a/cpp/subprojects/packagefiles/tcb-span/meson.build +++ b/cpp/subprojects/packagefiles/tcb-span/meson.build @@ -17,6 +17,8 @@ project('tcb-span', 'cpp') -tcb_span_dep = declare_dependency(include_directories: include_directories('include')) +tcb_span_dep = declare_dependency( + include_directories: include_directories('include') +) meson.override_dependency('tcb_span', tcb_span_dep) From 01d10b35b7282f3524963580d416c077db2b33a5 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 27 Jan 2026 00:04:48 -0600 Subject: [PATCH 20/36] typo --- cpp/subprojects/packagefiles/magic-enum/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/subprojects/packagefiles/magic-enum/meson.build b/cpp/subprojects/packagefiles/magic-enum/meson.build index cdf68711c5b..5c8f9f68005 100644 --- a/cpp/subprojects/packagefiles/magic-enum/meson.build +++ b/cpp/subprojects/packagefiles/magic-enum/meson.build @@ -18,7 +18,7 @@ project('magic-enum', 'cpp') magic_enum_dep = declare_dependency( - include_directories: include_directories('include' + include_directories: include_directories('include') ) meson.override_dependency('magic_enum_header_only', magic_enum_dep) From 05083e9f4cd202c012cad8748f7050616bf542f0 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 27 Jan 2026 01:02:18 -0600 Subject: [PATCH 21/36] Benchmark lib failing --- cpp/src/parquet/meson.build | 7 ++++--- cpp/subprojects/packagefiles/magic-enum/meson.build | 2 +- cpp/subprojects/packagefiles/tcb-span/meson.build | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 3dd88f42d57..0544903552f 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -147,17 +147,18 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() 'CMAKE_BUILD_TYPE': build_type, 'BUILD_SHARED_LIBS': 'ON', 'BUILD_TESTING': 'OFF', + 'BUILD_BENCHMARKS': 'OFF', 'CMAKE_LIBRARY_OUTPUT_DIRECTORY': build_output_dir, 'CMAKE_RUNTIME_OUTPUT_DIRECTORY': build_output_dir, 'CMAKE_ARCHIVE_OUTPUT_DIRECTORY': build_output_dir, - } + }, ) dbps_proj = cmake.subproject('dbps_agent', options: dbps_opts) dbps_interface_dep = declare_dependency( include_directories: include_directories( '../../subprojects/DataBatchProtectionService-6206fb0e27556a0df9160364caa3819e4af3fe0f/src/common', - ) + ), ) parquet_deps += [dbps_interface_dep, tcb_span_dep, magic_enum_dep] @@ -172,7 +173,7 @@ parquet_lib = library( parquet_dep = declare_dependency( link_with: parquet_lib, - dependencies: parquet_deps + dependencies: parquet_deps, ) subdir('api') diff --git a/cpp/subprojects/packagefiles/magic-enum/meson.build b/cpp/subprojects/packagefiles/magic-enum/meson.build index 5c8f9f68005..fd877d94704 100644 --- a/cpp/subprojects/packagefiles/magic-enum/meson.build +++ b/cpp/subprojects/packagefiles/magic-enum/meson.build @@ -18,7 +18,7 @@ project('magic-enum', 'cpp') magic_enum_dep = declare_dependency( - include_directories: include_directories('include') + include_directories: include_directories('include'), ) meson.override_dependency('magic_enum_header_only', magic_enum_dep) diff --git a/cpp/subprojects/packagefiles/tcb-span/meson.build b/cpp/subprojects/packagefiles/tcb-span/meson.build index 6e9d99e0cab..99966eba78e 100644 --- a/cpp/subprojects/packagefiles/tcb-span/meson.build +++ b/cpp/subprojects/packagefiles/tcb-span/meson.build @@ -18,7 +18,7 @@ project('tcb-span', 'cpp') tcb_span_dep = declare_dependency( - include_directories: include_directories('include') + include_directories: include_directories('include'), ) meson.override_dependency('tcb_span', tcb_span_dep) From 6117f47aabf99af7487637b2ff1c59a2f45028e6 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 27 Jan 2026 01:48:02 -0600 Subject: [PATCH 22/36] Last attempt today --- cpp/src/parquet/meson.build | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 0544903552f..46afcfcfea9 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -151,6 +151,15 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() 'CMAKE_LIBRARY_OUTPUT_DIRECTORY': build_output_dir, 'CMAKE_RUNTIME_OUTPUT_DIRECTORY': build_output_dir, 'CMAKE_ARCHIVE_OUTPUT_DIRECTORY': build_output_dir, + 'CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG': build_output_dir, + 'CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE': build_output_dir, + 'CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG': build_output_dir, + 'CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE': build_output_dir, + 'CMAKE_ARCHIVE_OUTPUT_DIRECTORY_DEBUG': build_output_dir, + 'CMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELEASE': build_output_dir, + 'CMAKE_BUILD_WITH_INSTALL_RPATH': 'ON', + 'CMAKE_INSTALL_RPATH': build_output_dir, + 'CMAKE_INSTALL_RPATH_USE_LINK_PATH': 'ON', }, ) From 653fe319bae67785540b2c85c5ec292ec54b7520 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 27 Jan 2026 08:54:11 -0600 Subject: [PATCH 23/36] Last attempt today --- cpp/src/parquet/meson.build | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 46afcfcfea9..88d4f16adb2 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -160,6 +160,11 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() 'CMAKE_BUILD_WITH_INSTALL_RPATH': 'ON', 'CMAKE_INSTALL_RPATH': build_output_dir, 'CMAKE_INSTALL_RPATH_USE_LINK_PATH': 'ON', + 'CMAKE_BUILD_RPATH': build_output_dir, + 'CMAKE_BUILD_WITH_INSTALL_RPATH': 'ON', + 'CMAKE_INSTALL_RPATH': build_output_dir, + 'CMAKE_INSTALL_RPATH_USE_LINK_PATH': 'ON', + 'CMAKE_BUILD_RPATH_USE_ORIGIN': 'ON', }, ) From f0dcaa1fd042136dd11e9f02d308fcb20ef0d031 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 27 Jan 2026 09:13:48 -0600 Subject: [PATCH 24/36] Linker errors --- cpp/src/parquet/meson.build | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 88d4f16adb2..e1e87ecbaac 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -161,7 +161,6 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() 'CMAKE_INSTALL_RPATH': build_output_dir, 'CMAKE_INSTALL_RPATH_USE_LINK_PATH': 'ON', 'CMAKE_BUILD_RPATH': build_output_dir, - 'CMAKE_BUILD_WITH_INSTALL_RPATH': 'ON', 'CMAKE_INSTALL_RPATH': build_output_dir, 'CMAKE_INSTALL_RPATH_USE_LINK_PATH': 'ON', 'CMAKE_BUILD_RPATH_USE_ORIGIN': 'ON', From 03ec9df7ff25dc0a4177cc723b877cbc0fb9a58e Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 27 Jan 2026 09:20:09 -0600 Subject: [PATCH 25/36] Linker errors --- cpp/src/parquet/meson.build | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index e1e87ecbaac..bb14538ee44 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -161,8 +161,6 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() 'CMAKE_INSTALL_RPATH': build_output_dir, 'CMAKE_INSTALL_RPATH_USE_LINK_PATH': 'ON', 'CMAKE_BUILD_RPATH': build_output_dir, - 'CMAKE_INSTALL_RPATH': build_output_dir, - 'CMAKE_INSTALL_RPATH_USE_LINK_PATH': 'ON', 'CMAKE_BUILD_RPATH_USE_ORIGIN': 'ON', }, ) From 5e45d220f29de9ea651999692272b8665d0d909b Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 27 Jan 2026 10:03:19 -0600 Subject: [PATCH 26/36] Linker errors --- cpp/src/parquet/meson.build | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index bb14538ee44..88f2b58b851 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -160,7 +160,6 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() 'CMAKE_BUILD_WITH_INSTALL_RPATH': 'ON', 'CMAKE_INSTALL_RPATH': build_output_dir, 'CMAKE_INSTALL_RPATH_USE_LINK_PATH': 'ON', - 'CMAKE_BUILD_RPATH': build_output_dir, 'CMAKE_BUILD_RPATH_USE_ORIGIN': 'ON', }, ) From 5a2c6807d1cdb4cddb47cc52f2862ade181d34f9 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 27 Jan 2026 10:37:34 -0600 Subject: [PATCH 27/36] Linker errors --- cpp/src/parquet/meson.build | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 88f2b58b851..f170c3c8c18 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -148,17 +148,20 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() 'BUILD_SHARED_LIBS': 'ON', 'BUILD_TESTING': 'OFF', 'BUILD_BENCHMARKS': 'OFF', - 'CMAKE_LIBRARY_OUTPUT_DIRECTORY': build_output_dir, - 'CMAKE_RUNTIME_OUTPUT_DIRECTORY': build_output_dir, - 'CMAKE_ARCHIVE_OUTPUT_DIRECTORY': build_output_dir, - 'CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG': build_output_dir, - 'CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE': build_output_dir, - 'CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG': build_output_dir, - 'CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE': build_output_dir, - 'CMAKE_ARCHIVE_OUTPUT_DIRECTORY_DEBUG': build_output_dir, - 'CMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELEASE': build_output_dir, + # 'CMAKE_LIBRARY_OUTPUT_DIRECTORY': build_output_dir, + #'CMAKE_RUNTIME_OUTPUT_DIRECTORY': build_output_dir, + #'CMAKE_ARCHIVE_OUTPUT_DIRECTORY': build_output_dir, + #'CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG': build_output_dir, + #'CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE': build_output_dir, + #'CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG': build_output_dir, + #'CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE': build_output_dir, + #'CMAKE_ARCHIVE_OUTPUT_DIRECTORY_DEBUG': build_output_dir, + #'CMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELEASE': build_output_dir, + #'CMAKE_BUILD_WITH_INSTALL_RPATH': 'ON', + #'CMAKE_INSTALL_RPATH': build_output_dir, + #'CMAKE_INSTALL_RPATH_USE_LINK_PATH': 'ON', + #'CMAKE_BUILD_RPATH_USE_ORIGIN': 'ON', 'CMAKE_BUILD_WITH_INSTALL_RPATH': 'ON', - 'CMAKE_INSTALL_RPATH': build_output_dir, 'CMAKE_INSTALL_RPATH_USE_LINK_PATH': 'ON', 'CMAKE_BUILD_RPATH_USE_ORIGIN': 'ON', }, From 0449773f09dd2ece9b78cfdee23f14306f990519 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 27 Jan 2026 10:48:40 -0600 Subject: [PATCH 28/36] Linker errors --- cpp/src/parquet/meson.build | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index f170c3c8c18..f0acc967ce3 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -148,6 +148,10 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() 'BUILD_SHARED_LIBS': 'ON', 'BUILD_TESTING': 'OFF', 'BUILD_BENCHMARKS': 'OFF', + 'GTEST_BUILD_TESTS': 'OFF', + 'GTEST_BUILD_SAMPLES': 'OFF', + 'BENCHMARK_ENABLE_TESTING': 'OFF', + 'BENCHMARK_ENABLE_INSTALL': 'OFF', # 'CMAKE_LIBRARY_OUTPUT_DIRECTORY': build_output_dir, #'CMAKE_RUNTIME_OUTPUT_DIRECTORY': build_output_dir, #'CMAKE_ARCHIVE_OUTPUT_DIRECTORY': build_output_dir, From 3df6c299b5f244c536f9f97969e4e2ea703114de Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 27 Jan 2026 11:34:57 -0600 Subject: [PATCH 29/36] Trying to fix cycle --- cpp/subprojects/dbps_agent.wrap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/subprojects/dbps_agent.wrap b/cpp/subprojects/dbps_agent.wrap index ac8c37bf1cb..a9cf66008de 100644 --- a/cpp/subprojects/dbps_agent.wrap +++ b/cpp/subprojects/dbps_agent.wrap @@ -20,4 +20,4 @@ directory = DataBatchProtectionService-6206fb0e27556a0df9160364caa3819e4af3fe0f source_url = https://github.com/protegrity/DataBatchProtectionService/archive/6206fb0e27556a0df9160364caa3819e4af3fe0f.tar.gz source_filename = dbps_agent-6206fb0e27556a0df9160364caa3819e4af3fe0f.tar.gz source_hash = 9c95a1fec0c9851867a776c3241d3feb59b07bd7a50e653d6214e07a8ad62419 - +patch_directory = dbps_agent From f33d31e8688aa91900dfea17f6bfa7dec29f3c99 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 27 Jan 2026 11:54:34 -0600 Subject: [PATCH 30/36] Patching DBPS build file --- cpp/src/parquet/meson.build | 2 +- .../dbps_agent/CMakeLists.txt.patch | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 cpp/subprojects/packagefiles/dbps_agent/CMakeLists.txt.patch diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index f0acc967ce3..cac7a26b080 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -146,7 +146,7 @@ if needs_parquet_encryption or get_option('parquet_require_encryption').auto() { 'CMAKE_BUILD_TYPE': build_type, 'BUILD_SHARED_LIBS': 'ON', - 'BUILD_TESTING': 'OFF', + 'BUILD_TESTS': 'OFF', 'BUILD_BENCHMARKS': 'OFF', 'GTEST_BUILD_TESTS': 'OFF', 'GTEST_BUILD_SAMPLES': 'OFF', diff --git a/cpp/subprojects/packagefiles/dbps_agent/CMakeLists.txt.patch b/cpp/subprojects/packagefiles/dbps_agent/CMakeLists.txt.patch new file mode 100644 index 00000000000..5de49c10b51 --- /dev/null +++ b/cpp/subprojects/packagefiles/dbps_agent/CMakeLists.txt.patch @@ -0,0 +1,30 @@ +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -520,15 +520,19 @@ endif() + + # All target (everything) + add_custom_target(all_targets + DEPENDS + libraries + executables + COMMENT "Building all targets" + ) + + # Add tests to all_targets if enabled + if(BUILD_TESTS) +- add_dependencies(all_targets tests) ++ # Only add test dependencies when tests target actually exists ++ # This prevents Meson from detecting cycles when BUILD_TESTS is OFF ++ if(TARGET tests) ++ add_dependencies(all_targets tests) ++ endif() + endif() + + # Add client shared library to all_targets if enabled + if(BUILD_SHARED_LIBS) +- add_dependencies(all_targets shared_libraries) ++ # Only add shared library dependencies when shared_libraries target exists ++ if(TARGET shared_libraries) ++ add_dependencies(all_targets shared_libraries) ++ endif() + endif() \ No newline at end of file From 7c0eb3722b89c6f668f596917084f7b9a768fca8 Mon Sep 17 00:00:00 2001 From: Sofia Brenes Date: Tue, 27 Jan 2026 12:30:09 -0600 Subject: [PATCH 31/36] Patching DBPS build file --- .../packagefiles/dbps_agent/CMakeLists.txt.patch | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cpp/subprojects/packagefiles/dbps_agent/CMakeLists.txt.patch b/cpp/subprojects/packagefiles/dbps_agent/CMakeLists.txt.patch index 5de49c10b51..a06c90233b3 100644 --- a/cpp/subprojects/packagefiles/dbps_agent/CMakeLists.txt.patch +++ b/cpp/subprojects/packagefiles/dbps_agent/CMakeLists.txt.patch @@ -1,5 +1,20 @@ --- a/CMakeLists.txt +++ b/CMakeLists.txt +@@ -10,8 +10,10 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) + # Build options + option(BUILD_TESTS "Build test executables" ON) + option(BUILD_SHARED_LIBS "Build shared libraries" ON) + + # Enable CTest and GoogleTest integration +-enable_testing() +-include(CTest) ++if(BUILD_TESTS) ++ enable_testing() ++ include(CTest) ++endif() + + # ============================================================================= + # Dependencies @@ -520,15 +520,19 @@ endif() # All target (everything) From aa9b0e0d11873c1b96fabe04dd44df00049d3e2d Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Tue, 27 Jan 2026 17:18:50 -0600 Subject: [PATCH 32/36] Importing DBPS as headers-only in meson --- cpp/src/arrow/meson.build | 6 +-- cpp/src/parquet/meson.build | 88 ++++++++++++++----------------------- 2 files changed, 35 insertions(+), 59 deletions(-) diff --git a/cpp/src/arrow/meson.build b/cpp/src/arrow/meson.build index 5590ba41c91..5455e0e6a72 100644 --- a/cpp/src/arrow/meson.build +++ b/cpp/src/arrow/meson.build @@ -391,7 +391,9 @@ if needs_filesystem {'ENV{AZURE_SDK_DISABLE_AUTO_VCPKG}': 'TRUE'}, {'WARNINGS_AS_ERRORS': 'FALSE'}, ) - azure_opt.append_compile_args('cpp', '-fPIC') + if host_machine.system() != 'windows' + azure_opt.append_compile_args('cpp', '-fPIC') + endif azure_proj = cmake.subproject('azure', options: azure_opt) azure_dep = declare_dependency( @@ -628,13 +630,11 @@ if needs_testing gtest_dep = dependency('gtest') gtest_main_dep = dependency('gtest_main') - gtest_dep = dependency('gtest') gmock_dep = dependency('gmock') else filesystem_dep = disabler() gtest_dep = disabler() gtest_main_dep = disabler() - gtest_dep = disabler() gmock_dep = disabler() endif diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index cac7a26b080..a3ca4668993 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -119,68 +119,44 @@ if openssl_dep.found() 'encryption/local_wrap_kms_client.cc', 'encryption/openssl_internal.cc', ) -else - parquet_srcs += files('encryption/aes_encryption_nossl.cc') -endif - -if needs_parquet_encryption or get_option('parquet_require_encryption').auto() - tcb_span_dep = dependency('tcb_span', fallback: ['tcb-span', 'tcb_span_dep']) - magic_enum_dep = dependency( - 'magic_enum_header_only', - fallback: ['magic-enum', 'magic_enum_dep'], - ) - cmake = import('cmake') - dbps_opts = cmake.subproject_options() + # External DBPA integration and its header-only deps are only relevant when + # encryption support is enabled (i.e., OpenSSL is available). + if needs_parquet_encryption or get_option('parquet_require_encryption').auto() + tcb_span_dep = dependency('tcb_span', fallback: ['tcb-span', 'tcb_span_dep']) + magic_enum_dep = dependency( + 'magic_enum_header_only', + fallback: ['magic-enum', 'magic_enum_dep'], + ) - build_type = get_option('buildtype') - if build_type == 'plain' - build_type = 'Release' - elif build_type == 'debug' - build_type = 'Debug' - endif + # DBPS interface is header-only (dbpa_interface.h and friends). + # + # IMPORTANT: Do not use Meson's CMake interpreter for DBPS here. DBPS' CMake + # target graph (CTest/CDash targets, generator expressions, etc.) has proven + # incompatible with Meson's CMake dependency extractor and can fail Meson + # configuration with "Cycle in CMake inputs/dependencies detected". + # + # Instead we use a tiny Meson wrapper (cpp/subprojects/packagefiles/dbps_agent) + # to expose the header-only dependency, and (optionally) build DBPS shared + # libraries via a build-time custom target that calls the real CMake. + dbps_sp = subproject('dbps_agent') + dbps_interface_dep = dbps_sp.get_variable('dbps_interface_dep') - build_output_dir = meson.current_build_dir() + parquet_deps += [dbps_interface_dep, tcb_span_dep, magic_enum_dep] - dbps_opts.add_cmake_defines( - { - 'CMAKE_BUILD_TYPE': build_type, - 'BUILD_SHARED_LIBS': 'ON', - 'BUILD_TESTS': 'OFF', - 'BUILD_BENCHMARKS': 'OFF', - 'GTEST_BUILD_TESTS': 'OFF', - 'GTEST_BUILD_SAMPLES': 'OFF', - 'BENCHMARK_ENABLE_TESTING': 'OFF', - 'BENCHMARK_ENABLE_INSTALL': 'OFF', - # 'CMAKE_LIBRARY_OUTPUT_DIRECTORY': build_output_dir, - #'CMAKE_RUNTIME_OUTPUT_DIRECTORY': build_output_dir, - #'CMAKE_ARCHIVE_OUTPUT_DIRECTORY': build_output_dir, - #'CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG': build_output_dir, - #'CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE': build_output_dir, - #'CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG': build_output_dir, - #'CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE': build_output_dir, - #'CMAKE_ARCHIVE_OUTPUT_DIRECTORY_DEBUG': build_output_dir, - #'CMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELEASE': build_output_dir, - #'CMAKE_BUILD_WITH_INSTALL_RPATH': 'ON', - #'CMAKE_INSTALL_RPATH': build_output_dir, - #'CMAKE_INSTALL_RPATH_USE_LINK_PATH': 'ON', - #'CMAKE_BUILD_RPATH_USE_ORIGIN': 'ON', - 'CMAKE_BUILD_WITH_INSTALL_RPATH': 'ON', - 'CMAKE_INSTALL_RPATH_USE_LINK_PATH': 'ON', - 'CMAKE_BUILD_RPATH_USE_ORIGIN': 'ON', - }, - ) - - dbps_proj = cmake.subproject('dbps_agent', options: dbps_opts) - dbps_interface_dep = declare_dependency( - include_directories: include_directories( - '../../subprojects/DataBatchProtectionService-6206fb0e27556a0df9160364caa3819e4af3fe0f/src/common', - ), - ) - - parquet_deps += [dbps_interface_dep, tcb_span_dep, magic_enum_dep] + if get_option('parquet_build_dbps_libs').enabled() + warning( + 'Meson does not build DBPS shared libraries (parquet_build_dbps_libs is a no-op). ' + + 'Provide your own agent shared library and set configuration_properties["agent_library_path"], ' + + 'or build DBPS via its CMake build separately.', + ) + endif + endif +else + parquet_srcs += files('encryption/aes_encryption_nossl.cc') endif + parquet_lib = library( 'arrow-parquet', sources: parquet_srcs, From bd0ac7a1d582775a10a3ec22d2bc343ba1ff6057 Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Tue, 27 Jan 2026 21:48:46 -0600 Subject: [PATCH 33/36] fix meson --- cpp/src/arrow/meson.build | 8 +++++++- cpp/src/parquet/meson.build | 21 +++++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/meson.build b/cpp/src/arrow/meson.build index 5455e0e6a72..8a0e870d6c7 100644 --- a/cpp/src/arrow/meson.build +++ b/cpp/src/arrow/meson.build @@ -385,12 +385,14 @@ if needs_filesystem {'BUILD_PERFORMANCE_TESTS': 'FALSE'}, {'BUILD_SAMPLES': 'FALSE'}, {'BUILD_TESTING': 'FALSE'}, - {'BUILD_WINDOWS_UWP': 'TRUE'}, {'CMAKE_UNITY_BUILD': 'FALSE'}, {'DISABLE_AZURE_CORE_OPENTELEMETRY': 'TRUE'}, {'ENV{AZURE_SDK_DISABLE_AUTO_VCPKG}': 'TRUE'}, {'WARNINGS_AS_ERRORS': 'FALSE'}, ) + if host_machine.system() == 'windows' + azure_opt.add_cmake_defines({'BUILD_WINDOWS_UWP': 'TRUE'}) + endif if host_machine.system() != 'windows' azure_opt.append_compile_args('cpp', '-fPIC') endif @@ -623,6 +625,10 @@ if needs_testing boost_opt = cmake.subproject_options() boost_opt.add_cmake_defines( {'BOOST_INCLUDE_LIBRARIES': 'filesystem;system'}, + # Keep Boost's CMake graph minimal for Meson's CMake introspection. + {'BUILD_TESTING': 'OFF'}, + {'BOOST_ENABLE_TESTING': 'OFF'}, + {'CMAKE_UNITY_BUILD': 'OFF'}, ) boost_proj = cmake.subproject('boost', options: boost_opt) filesystem_dep = boost_proj.dependency('boost_filesystem') diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index a3ca4668993..4d3f0efe8a6 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -65,6 +65,8 @@ if not thrift_dep.found() { 'BUILD_COMPILER': 'OFF', 'BUILD_EXAMPLES': 'OFF', + 'BUILD_TESTING': 'OFF', + 'BUILD_TESTS': 'OFF', 'BUILD_TUTORIALS': 'OFF', 'CMAKE_UNITY_BUILD': 'OFF', 'WITH_AS3': 'OFF', @@ -157,11 +159,22 @@ else endif +# Parquet's CMake build uses explicit export macros and (on ELF) a version script +# to control symbol visibility. Meson doesn't currently replicate that machinery. +# With Meson's default hidden visibility, some non-exported-but-test-used symbols +# (e.g. EncodingProperties, IsParquetCipherSupported) are not linkable from test +# executables. When building tests/benchmarks, relax visibility to avoid link +# failures in Meson CI. +parquet_symbol_visibility = 'inlineshidden' +if needs_testing + parquet_symbol_visibility = 'default' +endif + parquet_lib = library( 'arrow-parquet', sources: parquet_srcs, dependencies: parquet_deps, - gnu_symbol_visibility: 'inlineshidden', + gnu_symbol_visibility: parquet_symbol_visibility, ) parquet_dep = declare_dependency( @@ -271,6 +284,7 @@ parquet_tests = { 'arrow/arrow_reader_writer_test.cc', 'arrow/arrow_statistics_test.cc', 'arrow/variant_test.cc', + 'encryption/external/test_utils.cc', ), }, 'arrow-internals-test': { @@ -285,7 +299,9 @@ parquet_tests = { 'arrow/arrow_schema_test.cc', ), }, - 'file_deserialize_test': {'sources': files('file_deserialize_test.cc')}, + 'file_deserialize_test': { + 'sources': files('file_deserialize_test.cc', 'encryption/external/test_utils.cc'), + }, 'schema_test': {'sources': files('schema_test.cc')}, } @@ -302,6 +318,7 @@ if needs_parquet_encryption 'encryption/properties_test.cc', 'encryption/read_configurations_test.cc', 'encryption/test_encryption_util.cc', + 'encryption/test_in_memory_kms.cc', 'encryption/write_configurations_test.cc', ), }, From 7334036442664f6dfd8d8f298462368d4b3c69d4 Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Tue, 27 Jan 2026 22:43:34 -0600 Subject: [PATCH 34/36] making meson build test shared libs for encryption/external --- cpp/src/parquet/meson.build | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 4d3f0efe8a6..456c7247235 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -153,6 +153,24 @@ if openssl_dep.found() 'or build DBPS via its CMake build separately.', ) endif + + # Build the in-tree DBPA test agent shared library used by external encryption tests. + # CMake builds this as `DBPATestAgent` when ARROW_TESTING is enabled; Meson needs an + # equivalent target so tests can dlopen() `libDBPATestAgent.so`. + # + # Keep it Meson-native (no CMake), and place the output next to parquet test + # executables so `TestUtils::GetTestLibraryPath()` can find it via the executable dir. + dbpa_test_agent_lib = disabler() + if needs_testing + dbpa_test_agent_lib = shared_library( + 'DBPATestAgent', + sources: files('encryption/external/dbpa_test_agent.cc'), + include_directories: include_directories('..'), + dependencies: [arrow_dep, magic_enum_dep, tcb_span_dep, dbps_interface_dep], + install: false, + gnu_symbol_visibility: 'default', + ) + endif endif else parquet_srcs += files('encryption/aes_encryption_nossl.cc') @@ -364,7 +382,9 @@ foreach key, val : parquet_tests sources: val['sources'] + files('test_util.cc'), dependencies: parquet_test_dep, ) - test(test_name, exc) + # Ensure the DBPA test agent is built before running any tests that may load it. + # (No-op when dbpa_test_agent_lib is disabled/unset.) + test(test_name, exc, depends: dbpa_test_agent_lib) endforeach parquet_benchmarks = { From 31a30c54ee8e658ee9411ed7e5172c01b7bfc28f Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Wed, 28 Jan 2026 09:31:38 -0600 Subject: [PATCH 35/36] improved logging --- .../parquet/encryption/external/test_utils.cc | 28 ++++++++++++++++++- .../encryption/external_dbpa_encryption.cc | 13 +++++++-- cpp/src/parquet/meson.build | 28 +++++++++++++++++-- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/cpp/src/parquet/encryption/external/test_utils.cc b/cpp/src/parquet/encryption/external/test_utils.cc index 9638a2607f3..e46099b79e0 100644 --- a/cpp/src/parquet/encryption/external/test_utils.cc +++ b/cpp/src/parquet/encryption/external/test_utils.cc @@ -20,6 +20,7 @@ #include #include #include +#include #ifdef __APPLE__ # include @@ -57,6 +58,21 @@ std::string TestUtils::GetExecutableDirectory() { } std::string TestUtils::GetTestLibraryPath() { + // Strong override: reuse the same env var as the Python tooling + // (`python/scripts/base_app.py`): DBPA_LIBRARY_PATH. + // + // This allows CI/build systems to provide the exact path to the DBPA agent shared + // library, avoiding reliance on executable-path heuristics or current working directory. + const char* explicit_path = std::getenv("DBPA_LIBRARY_PATH"); + if (explicit_path && explicit_path[0]) { + std::string p(explicit_path); + if (std::filesystem::exists(p)) { + return p; + } + throw std::runtime_error("DBPA_LIBRARY_PATH is set but the file does not exist: " + + p); + } + // Check for environment variable to override the executable directory const char* cwd_override = std::getenv("PARQUET_TEST_LIBRARY_CWD"); std::string base_path; @@ -83,7 +99,17 @@ std::string TestUtils::GetTestLibraryPath() { } } - throw std::runtime_error("Could not find library"); + // Provide a detailed error to make CI failures diagnosable. + std::string msg = "Could not find DBPA test agent library. Tried:\n"; + for (const auto& filename : possible_filenames) { + for (const auto& directory : possible_directories) { + msg += " - " + (directory + filename) + "\n"; + } + } + msg += "PARQUET_TEST_LIBRARY_CWD="; + msg += (cwd_override && cwd_override[0]) ? cwd_override : ""; + msg += "\n"; + throw std::runtime_error(msg); } } // namespace parquet::encryption::external::test diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption.cc b/cpp/src/parquet/encryption/external_dbpa_encryption.cc index 513020f00da..85e74cab002 100644 --- a/cpp/src/parquet/encryption/external_dbpa_encryption.cc +++ b/cpp/src/parquet/encryption/external_dbpa_encryption.cc @@ -62,8 +62,17 @@ std::unique_ptr LoadAndInitia // Step 1: Get path to the shared library auto it = configuration_properties.find(SHARED_LIBRARY_PATH_KEY); if (it == configuration_properties.end()) { - const auto msg = "Required configuration key '" + SHARED_LIBRARY_PATH_KEY + - "' not found in configuration_properties"; + std::string msg = "Required configuration key '" + SHARED_LIBRARY_PATH_KEY + + "' not found in configuration_properties. Present keys: "; + bool first = true; + for (const auto& kv : configuration_properties) { + if (!first) msg += ", "; + first = false; + msg += kv.first; + } + if (first) { + msg += ""; + } ARROW_LOG(ERROR) << msg; throw ParquetException(msg); } diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index 456c7247235..fb726ecd65f 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -91,6 +91,10 @@ endif parquet_deps = [arrow_dep, rapidjson_dep, thrift_dep] +# Default to no DBPA test-agent library (only built when encryption+testing are enabled). +dbpa_test_agent_lib = disabler() +dbpa_test_agent_path = '' + if needs_parquet_encryption or get_option('parquet_require_encryption').auto() openssl_dep = dependency('openssl', required: needs_parquet_encryption) else @@ -160,16 +164,19 @@ if openssl_dep.found() # # Keep it Meson-native (no CMake), and place the output next to parquet test # executables so `TestUtils::GetTestLibraryPath()` can find it via the executable dir. - dbpa_test_agent_lib = disabler() if needs_testing dbpa_test_agent_lib = shared_library( 'DBPATestAgent', sources: files('encryption/external/dbpa_test_agent.cc'), include_directories: include_directories('..'), - dependencies: [arrow_dep, magic_enum_dep, tcb_span_dep, dbps_interface_dep], + # Keep this test agent as self-contained as possible. It is dlopen()'d + # by tests, so avoid unnecessary runtime dependencies (e.g., libarrow.so) + # which can differ between Meson/CMake CI environments. + dependencies: [magic_enum_dep, tcb_span_dep, dbps_interface_dep], install: false, gnu_symbol_visibility: 'default', ) + dbpa_test_agent_path = dbpa_test_agent_lib.full_path() endif endif else @@ -384,7 +391,22 @@ foreach key, val : parquet_tests ) # Ensure the DBPA test agent is built before running any tests that may load it. # (No-op when dbpa_test_agent_lib is disabled/unset.) - test(test_name, exc, depends: dbpa_test_agent_lib) + test( + test_name, + exc, + depends: dbpa_test_agent_lib, + env: { + # Make DBPATestAgent lookup deterministic under Meson. Some CI setups may + # not allow /proc/self/exe resolution or run tests with unexpected cwd. + 'PARQUET_TEST_LIBRARY_CWD': meson.current_build_dir(), + # Reuse the standard env var used by Python tooling (`base_app.py`). + # Prefer this in C++ as well. + 'DBPA_LIBRARY_PATH': dbpa_test_agent_path, + # Make DBPA-related code emit useful logs in CI (opt-in via env). + # This helps debug why Meson runs see unexpected configuration_properties. + 'PARQUET_DBPA_LOG_LEVEL': 'DEBUG', + }, + ) endforeach parquet_benchmarks = { From 3896539bb258629b2d810de3c35eefe5678dcba5 Mon Sep 17 00:00:00 2001 From: Marco Arguedas Date: Sun, 1 Feb 2026 18:18:47 -0600 Subject: [PATCH 36/36] additional printlns --- .../encryption/external_dbpa_encryption.cc | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cpp/src/parquet/encryption/external_dbpa_encryption.cc b/cpp/src/parquet/encryption/external_dbpa_encryption.cc index 85e74cab002..45de26a2cae 100644 --- a/cpp/src/parquet/encryption/external_dbpa_encryption.cc +++ b/cpp/src/parquet/encryption/external_dbpa_encryption.cc @@ -439,6 +439,19 @@ ExternalDBPAEncryptorAdapter* ExternalDBPAEncryptorAdapterFactory::GetEncryptor( auto app_context = external_file_encryption_properties->app_context(); auto connection_config_for_algorithm = configuration_properties.at(algorithm); + if (::arrow::util::ArrowLog::IsLevelEnabled( + ::arrow::util::ArrowLogLevel::ARROW_DEBUG)) { + ARROW_LOG(DEBUG) << "ExternalDBPAEncryptorAdapterFactory::GetEncryptor - " + "selected configuration_properties for EXTERNAL_DBPA_V1:"; + if (connection_config_for_algorithm.empty()) { + ARROW_LOG(DEBUG) << " "; + } else { + for (const auto& [k, v] : connection_config_for_algorithm) { + ARROW_LOG(DEBUG) << " [" << k << "]: [" << v << "]"; + } + } + } + std::string key_id; try { auto key_metadata = @@ -668,6 +681,19 @@ std::unique_ptr ExternalDBPADecryptorAdapterFactory::GetDecr auto connection_config_for_algorithm = configuration_properties.at(algorithm); auto key_value_metadata = column_chunk_metadata->key_value_metadata(); + if (::arrow::util::ArrowLog::IsLevelEnabled( + ::arrow::util::ArrowLogLevel::ARROW_DEBUG)) { + ARROW_LOG(DEBUG) << "ExternalDBPADecryptorAdapterFactory::GetDecryptor - " + "selected configuration_properties for EXTERNAL_DBPA_V1:"; + if (connection_config_for_algorithm.empty()) { + ARROW_LOG(DEBUG) << " "; + } else { + for (const auto& [k, v] : connection_config_for_algorithm) { + ARROW_LOG(DEBUG) << " [" << k << "]: [" << v << "]"; + } + } + } + std::string key_id; try { auto key_metadata = KeyMetadata::Parse(crypto_metadata->key_metadata());