Skip to content

Commit de9f1e7

Browse files
authored
feat(python/adbc_driver_manager): Update python driver_manager to load manifests (#3018)
Adding a new function `AdbcDriverManagerDatabaseSetLoadFlags` so that bindings can set the load flags into the Driver Manager's internal `TempDatabase` to control the functionality of when to look for and load manifests. This also changes the Python driver_manager to load manifests by default but allows passing a `load_flags` option to manually pass in load flags to control the behavior. Also adding a new `driver_example_manifest.py` recipe to the driver_example docs.
1 parent 0000e8d commit de9f1e7

File tree

14 files changed

+268
-9
lines changed

14 files changed

+268
-9
lines changed

.gitattributes

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,11 @@
1717

1818
*.stdout.txt linguist-generated
1919
c/vendor/* linguist-vendored
20-
go/adbc/drivermgr/adbc.h linguist-generated
20+
go/adbc/drivermgr/arrow-adbc/adbc.h linguist-generated
21+
go/adbc/drivermgr/arrow-adbc/adbc_driver_manager.h linguist-generated
2122
go/adbc/drivermgr/adbc_driver_manager.cc linguist-generated
23+
go/adbc/drivermgr/current_arch.h linguist-generated
24+
go/adbc/pkg/bigquery/* linguist-generated
2225
go/adbc/pkg/flightsql/* linguist-generated
2326
go/adbc/pkg/panicdummy/* linguist-generated
2427
go/adbc/pkg/snowflake/* linguist-generated

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ c/apidoc/objects.inv
6161
docs/example.gz
6262
docs/example1.dat
6363
docs/example3.dat
64+
docs/source/cpp/recipe_driver/driver_example.toml
6465
python/.eggs/
6566
python/doc/
6667
# Egg metadata

c/driver_manager/adbc_driver_manager.cc

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -983,6 +983,7 @@ struct TempDatabase {
983983
std::string driver;
984984
std::string entrypoint;
985985
AdbcDriverInitFunc init_func = nullptr;
986+
AdbcLoadFlags load_flags = ADBC_LOAD_FLAG_ALLOW_RELATIVE_PATHS;
986987
};
987988

988989
/// Temporary state while the database is being configured.
@@ -1341,6 +1342,19 @@ AdbcStatusCode AdbcDatabaseSetOptionDouble(struct AdbcDatabase* database, const
13411342
return ADBC_STATUS_OK;
13421343
}
13431344

1345+
AdbcStatusCode AdbcDriverManagerDatabaseSetLoadFlags(struct AdbcDatabase* database,
1346+
AdbcLoadFlags flags,
1347+
struct AdbcError* error) {
1348+
if (database->private_driver) {
1349+
SetError(error, "Cannot SetLoadFlags after AdbcDatabaseInit");
1350+
return ADBC_STATUS_INVALID_STATE;
1351+
}
1352+
1353+
TempDatabase* args = reinterpret_cast<TempDatabase*>(database->private_data);
1354+
args->load_flags = flags;
1355+
return ADBC_STATUS_OK;
1356+
}
1357+
13441358
AdbcStatusCode AdbcDriverManagerDatabaseSetInitFunc(struct AdbcDatabase* database,
13451359
AdbcDriverInitFunc init_func,
13461360
struct AdbcError* error) {
@@ -1376,11 +1390,12 @@ AdbcStatusCode AdbcDatabaseInit(struct AdbcDatabase* database, struct AdbcError*
13761390
status = AdbcLoadDriverFromInitFunc(args->init_func, ADBC_VERSION_1_1_0,
13771391
database->private_driver, error);
13781392
} else if (!args->entrypoint.empty()) {
1379-
status = AdbcLoadDriver(args->driver.c_str(), args->entrypoint.c_str(),
1380-
ADBC_VERSION_1_1_0, database->private_driver, error);
1393+
status = AdbcFindLoadDriver(args->driver.c_str(), args->entrypoint.c_str(),
1394+
ADBC_VERSION_1_1_0, args->load_flags,
1395+
database->private_driver, error);
13811396
} else {
1382-
status = AdbcLoadDriver(args->driver.c_str(), nullptr, ADBC_VERSION_1_1_0,
1383-
database->private_driver, error);
1397+
status = AdbcFindLoadDriver(args->driver.c_str(), nullptr, ADBC_VERSION_1_1_0,
1398+
args->load_flags, database->private_driver, error);
13841399
}
13851400

13861401
if (status != ADBC_STATUS_OK) {

c/include/arrow-adbc/adbc_driver_manager.h

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,21 @@ AdbcStatusCode AdbcDriverManagerDatabaseSetInitFunc(struct AdbcDatabase* databas
137137
AdbcDriverInitFunc init_func,
138138
struct AdbcError* error);
139139

140+
/// \brief Set the load flags for the driver manager.
141+
///
142+
/// This is an extension to the ADBC API. The driver manager shims
143+
/// the AdbcDatabase* functions to allow you to specify the
144+
/// driver/entrypoint dynamically. This function lets you set the
145+
/// load flags explicitly, for applications that can dynamically
146+
/// load drivers on their own.
147+
///
148+
/// If this function isn't called, the default load flags are just to
149+
/// allow relative paths, disallowing the lookups of manifests.
150+
ADBC_EXPORT
151+
AdbcStatusCode AdbcDriverManagerDatabaseSetLoadFlags(struct AdbcDatabase* database,
152+
AdbcLoadFlags flags,
153+
struct AdbcError* error);
154+
140155
/// \brief Get a human-friendly description of a status code.
141156
ADBC_EXPORT
142157
const char* AdbcStatusCodeMessage(AdbcStatusCode code);

docs/source/cpp/driver_example.rst

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,3 +45,13 @@ package.
4545
data.frame(col = 1:3) |> write_adbc(con, "example.arrows")
4646
con |> read_adbc("SELECT * FROM example.arrows") |> as.data.frame()
4747
unlink("example.arrows")
48+
49+
Driver Manifests
50+
================
51+
52+
.. recipe:: recipe_driver/driver_example_manifest.py
53+
54+
Driver manifests can provide an easier way to install and manage ADBC drivers,
55+
via TOML files that describe some metadata along with the path to the driver
56+
shared library. The driver manager can read these manifests to locate and load
57+
drivers dynamically.

docs/source/cpp/recipe_driver/CMakeLists.txt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,14 @@ target_link_libraries(driver_example PRIVATE adbc_driver_framework
5151

5252
install(TARGETS driver_example)
5353

54+
include(./get_arch.cmake)
55+
cmake_path(SET DRIVER_PATH "${CMAKE_INSTALL_PREFIX}")
56+
cmake_path(APPEND DRIVER_PATH "${CMAKE_INSTALL_LIBDIR}"
57+
"${CMAKE_SHARED_LIBRARY_PREFIX}driver_example${CMAKE_SHARED_LIBRARY_SUFFIX}")
58+
59+
configure_file(driver_example.toml.in "${CMAKE_CURRENT_SOURCE_DIR}/driver_example.toml"
60+
@ONLY)
61+
5462
if(ADBC_DRIVER_EXAMPLE_BUILD_TESTS)
5563
fetchcontent_declare(googletest
5664
URL https://github.com/google/googletest/archive/refs/tags/v1.15.1.tar.gz
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name = 'Driver Example'
19+
publisher = 'arrow-adbc-docs'
20+
license = 'Apache-2.0'
21+
version = '@CMAKE_PROJECT_VERSION@'
22+
source = 'recipe'
23+
24+
[ADBC]
25+
version = 'v1.1.0'
26+
27+
[Driver]
28+
[Driver.shared]
29+
@OS@_@ARCH@@EXTRA@ = '@DRIVER_PATH@'
Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# RECIPE STARTS HERE
19+
#: After verifying the basic driver functionality, we can use the
20+
#: ``adbc_driver_manager`` Python package's built-in dbapi implementation
21+
#: to expose a ready-to-go Pythonic database API. This is also useful for
22+
#: high-level testing!
23+
#:
24+
#: First, we'll import pathlib for a few path calculations and the
25+
#: ``adbc_driver_manager``'s ``dbapi`` module:
26+
from pathlib import Path
27+
28+
from adbc_driver_manager import dbapi
29+
30+
31+
#: Next, we'll define a ``connect()`` function that wraps ``dbapi.connect()``
32+
#: with the location the .toml manifest file which points to the shared library
33+
#: we built using ``cmake`` in the previous section.
34+
#: For the purposes of our tutorial, this will be in current directory.
35+
def connect(uri: str):
36+
# we can point to the manifest file directly
37+
manifest_file = Path(".") / "driver_example.toml"
38+
if manifest_file.exists():
39+
return dbapi.connect(
40+
driver=str(manifest_file.resolve()), db_kwargs={"uri": uri}
41+
)
42+
43+
# alternatively, it can look for the manifest file in the user's config
44+
# directory ($HOME/.config/adbc/driver_example.toml) or the system's
45+
# config directory (/etc/adbc/driver_example.toml)
46+
return dbapi.connect(driver="driver_example", db_kwargs={"uri": uri})
47+
48+
49+
#: Next, we can give our driver a go! The two pieces we implemented in the driver
50+
#: were the "bulk ingest" feature and "select all from", so let's see if it works!
51+
if __name__ == "__main__":
52+
import pyarrow
53+
54+
with connect(uri=Path(__file__).parent.as_uri()) as con:
55+
data = pyarrow.table({"col": [1, 2, 3]})
56+
with con.cursor() as cur:
57+
cur.adbc_ingest("example.arrows", data, mode="create")
58+
59+
with con.cursor() as cur:
60+
cur.execute("SELECT * FROM example.arrows")
61+
print(cur.fetchall())
62+
# Output: [(1,), (2,), (3,)]
63+
64+
(Path(__file__).parent / "example.arrows").unlink()

docs/source/cpp/recipe_driver/driver_example_manifest.py.stdout.txt

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
if(WIN32)
19+
set(OS "windows")
20+
elseif(APPLE)
21+
set(OS "osx")
22+
else()
23+
if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
24+
set(OS "freebsd")
25+
else()
26+
set(OS "linux")
27+
endif()
28+
endif()
29+
30+
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|AMD64)$")
31+
set(ARCH "amd64")
32+
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(i386|i486|i586|i686)$")
33+
set(ARCH "x86")
34+
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv8b|armv8l|aarch64|arm64|AARCH64|ARM64)$")
35+
set(ARCH "arm64")
36+
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv7l|armhf|arm)$")
37+
set(ARCH "arm")
38+
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc|ppc64|ppcle|ppc64le)$")
39+
set(ARCH "powerpc")
40+
elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64")
41+
set(ARCH "riscv")
42+
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(s390|s390x)$")
43+
set(ARCH "s390x")
44+
else()
45+
message(FATAL_ERROR "Unsupported architecture: ${CMAKE_SYSTEM_PROCESSOR}")
46+
endif()
47+
48+
if(NOT APPLE)
49+
include(CheckCSourceRuns)
50+
51+
check_c_source_runs([=[
52+
#include <stdlib.h>
53+
#if defined(__GLIBC__)
54+
#error "GLIBC detected"
55+
#elif defined(__MUSL__)
56+
int main(void) { return EXIT_SUCCESS; }
57+
#else
58+
#error "Neither GLIBC nor Musl detected"
59+
#endif
60+
]=]
61+
IS_MUSL)
62+
63+
if(MINGW)
64+
set(EXTRA "_mingw")
65+
elseif(IS_MUSL)
66+
set(EXTRA "_musl")
67+
endif()
68+
endif()

0 commit comments

Comments
 (0)