diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 243cd65351aa2f..931130994dab17 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -54,6 +54,9 @@ DEFINE_String(custom_config_dir, "${DORIS_HOME}/conf"); // Dir of jdbc drivers DEFINE_String(jdbc_drivers_dir, "${DORIS_HOME}/plugins/jdbc_drivers"); +// Dir of java udf +DEFINE_String(java_udf_dir, "${DORIS_HOME}/plugins/java_udf"); + // cluster id DEFINE_Int32(cluster_id, "-1"); // port on which BackendService is exported diff --git a/be/src/common/config.h b/be/src/common/config.h index 55591fc36c770a..2b3691c9186123 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -88,6 +88,9 @@ DECLARE_String(custom_config_dir); // Dir of jdbc drivers DECLARE_String(jdbc_drivers_dir); +// Dir of java udf +DECLARE_String(java_udf_dir); + // cluster id DECLARE_Int32(cluster_id); // port on which BackendService is exported diff --git a/be/src/runtime/user_function_cache.cpp b/be/src/runtime/user_function_cache.cpp index e058b76ad33326..5ec3c633d153bf 100644 --- a/be/src/runtime/user_function_cache.cpp +++ b/be/src/runtime/user_function_cache.cpp @@ -42,6 +42,7 @@ #include "runtime/exec_env.h" #include "util/dynamic_util.h" #include "util/md5.h" +#include "util/path_util.h" #include "util/string_util.h" namespace doris { @@ -271,10 +272,12 @@ Status UserFunctionCache::_download_lib(const std::string& url, return Status::InternalError("fail to open file"); } + std::string udf_path = + doris::path_util::get_real_plugin_url(url, doris::config::java_udf_dir, "java_udf"); Md5Digest digest; HttpClient client; int64_t file_size = 0; - RETURN_IF_ERROR(client.init(url)); + RETURN_IF_ERROR(client.init(udf_path)); Status status; auto download_cb = [&status, &tmp_file, &fp, &digest, &file_size](const void* data, size_t length) { @@ -382,4 +385,5 @@ std::vector UserFunctionCache::_split_string_by_checksum(const std: return result; } + } // namespace doris diff --git a/be/src/runtime/user_function_cache.h b/be/src/runtime/user_function_cache.h index 2537d9376ecd8d..93759c261e2b85 100644 --- a/be/src/runtime/user_function_cache.h +++ b/be/src/runtime/user_function_cache.h @@ -75,8 +75,6 @@ class UserFunctionCache { std::string _get_file_name_from_url(const std::string& url) const; std::vector _split_string_by_checksum(const std::string& file); - std::string _check_and_return_default_driver_url(const std::string& url); - private: std::string _lib_dir; void* _current_process_handle = nullptr; diff --git a/be/src/util/path_util.cpp b/be/src/util/path_util.cpp index e05b9371e49a51..4b6f6adcbed165 100644 --- a/be/src/util/path_util.cpp +++ b/be/src/util/path_util.cpp @@ -20,6 +20,10 @@ // Use the POSIX version of dirname(3). See `man 3 dirname` #include +#include +#include + +#include "common/config.h" #include "gutil/strings/split.h" #include "gutil/strings/strip.h" @@ -64,5 +68,47 @@ std::string file_extension(const string& path) { return pos == string::npos ? "" : file_name.substr(pos); } +std::string get_real_plugin_url(const std::string& url, const std::string& plugin_dir_config_value, + const std::string& plugin_dir_name, const std::string& doris_home) { + if (url.find(":/") == std::string::npos) { + return check_and_return_default_plugin_url(url, plugin_dir_config_value, plugin_dir_name, + doris_home); + } + return url; +} + +std::string check_and_return_default_plugin_url(const std::string& url, + const std::string& plugin_dir_config_value, + const std::string& plugin_dir_name, + const std::string& doris_home) { + std::string home_dir = doris_home; + if (home_dir.empty()) { + const char* env_home = std::getenv("DORIS_HOME"); + if (env_home) { + home_dir = std::string(env_home); + } else { + return "file://" + plugin_dir_config_value + "/" + url; + } + } + + std::string default_url = home_dir + "/plugins/" + plugin_dir_name; + std::string default_old_url = home_dir + "/" + plugin_dir_name; + + if (plugin_dir_config_value == default_url) { + // If true, which means user does not set `jdbc_drivers_dir` and use the default one. + // Because in 2.1.8, we change the default value of `jdbc_drivers_dir` + // from `DORIS_HOME/jdbc_drivers` to `DORIS_HOME/plugins/jdbc_drivers`, + // so we need to check the old default dir for compatibility. + std::filesystem::path file = default_url + "/" + url; + if (std::filesystem::exists(file)) { + return "file://" + default_url + "/" + url; + } else { + return "file://" + default_old_url + "/" + url; + } + } else { + return "file://" + plugin_dir_config_value + "/" + url; + } +} + } // namespace path_util } // namespace doris diff --git a/be/src/util/path_util.h b/be/src/util/path_util.h index ee68122f4faaf3..61231e6239dd0d 100644 --- a/be/src/util/path_util.h +++ b/be/src/util/path_util.h @@ -55,5 +55,21 @@ std::string base_name(const std::string& path); // NOTE: path can be either one file's full path or only file name std::string file_extension(const std::string& path); +// Get the real URL for plugins (e.g., JDBC drivers). If the URL doesn't contain ":/", +// it will be treated as a relative path and converted to a file:// URL using provided dirs. +// plugin_dir_config_value is the configured plugin dir; plugin_dir_name is the dir name (e.g. "jdbc_drivers"). +// doris_home is optional; if empty, will use DORIS_HOME environment variable. +std::string get_real_plugin_url(const std::string& url, const std::string& plugin_dir_config_value, + const std::string& plugin_dir_name, + const std::string& doris_home = ""); + +// Check and return the default plugin URL using provided directories. +// plugin_dir_config_value is the configured drivers dir; plugin_dir_name is dir name. +// doris_home is optional; if empty, will use DORIS_HOME environment variable. +std::string check_and_return_default_plugin_url(const std::string& url, + const std::string& plugin_dir_config_value, + const std::string& plugin_dir_name, + const std::string& doris_home = ""); + } // namespace path_util } // namespace doris diff --git a/be/src/vec/exec/vjdbc_connector.cpp b/be/src/vec/exec/vjdbc_connector.cpp index 17e4781a5a6fe8..d4cf31f4f78674 100644 --- a/be/src/vec/exec/vjdbc_connector.cpp +++ b/be/src/vec/exec/vjdbc_connector.cpp @@ -36,6 +36,7 @@ #include "runtime/types.h" #include "runtime/user_function_cache.h" #include "util/jni-util.h" +#include "util/path_util.h" #include "util/runtime_profile.h" #include "vec/columns/column_nullable.h" #include "vec/core/block.h" @@ -126,7 +127,8 @@ Status JdbcConnector::open(RuntimeState* state, bool read) { // Add a scoped cleanup jni reference object. This cleans up local refs made below. JniLocalFrame jni_frame; { - std::string driver_path = _get_real_url(_conn_param.driver_path); + std::string driver_path = doris::path_util::get_real_plugin_url( + _conn_param.driver_path, doris::config::jdbc_drivers_dir, "jdbc_drivers", ""); TJdbcExecutorCtorParams ctor_params; ctor_params.__set_statement(_sql_str); @@ -633,11 +635,4 @@ Status JdbcConnector::_get_java_table_type(JNIEnv* env, TOdbcTableType::type tab return Status::OK(); } -std::string JdbcConnector::_get_real_url(const std::string& url) { - if (url.find(":/") == std::string::npos) { - return "file://" + config::jdbc_drivers_dir + "/" + url; - } - return url; -} - } // namespace doris::vectorized diff --git a/be/src/vec/exec/vjdbc_connector.h b/be/src/vec/exec/vjdbc_connector.h index 8838ce5984d322..b514389b5da34d 100644 --- a/be/src/vec/exec/vjdbc_connector.h +++ b/be/src/vec/exec/vjdbc_connector.h @@ -141,8 +141,6 @@ class JdbcConnector : public TableConnector { Status _get_java_table_type(JNIEnv* env, TOdbcTableType::type table_type, jobject* java_enum_obj); - std::string _get_real_url(const std::string& url); - bool _closed = false; jclass _executor_factory_clazz; jclass _executor_clazz; diff --git a/be/test/util/path_util_test.cpp b/be/test/util/path_util_test.cpp index 8806febb133dd0..3ec95c5a1ac8a9 100644 --- a/be/test/util/path_util_test.cpp +++ b/be/test/util/path_util_test.cpp @@ -20,6 +20,9 @@ #include #include +#include +#include +#include #include #include @@ -99,4 +102,120 @@ TEST(TestPathUtil, file_extension_test) { EXPECT_EQ(".", path_util::file_extension("a.b.c.")); } +// Helpers for plugin URL tests +namespace { + +std::string create_temp_home() { + namespace fs = std::filesystem; + fs::path base = fs::temp_directory_path() / "doris_path_util_test"; + fs::create_directories(base); + // ensure unique subdir per test run + fs::path home = base / std::to_string(reinterpret_cast(&base)); + fs::create_directories(home); + return home.string(); +} + +void touch_file(const std::string& dir, const std::string& filename) { + namespace fs = std::filesystem; + fs::create_directories(dir); + std::ofstream ofs(fs::path(dir) / filename, std::ios::binary); + ofs << "x"; +} + +} // namespace + +TEST(TestPathUtil, get_real_plugin_url_absolute_passthrough) { + // absolute style URLs containing ":/" should be returned as-is + EXPECT_EQ("http://example.com/a.jar", + path_util::get_real_plugin_url("http://example.com/a.jar", "/any/dir", "jdbc_drivers", + "")); + EXPECT_EQ("file:///opt/driver/a.jar", + path_util::get_real_plugin_url("file:///opt/driver/a.jar", "/any/dir", "jdbc_drivers", + "")); +} + +TEST(TestPathUtil, check_and_return_default_plugin_url_prefers_new_default_when_exists) { + namespace fs = std::filesystem; + std::string home = create_temp_home(); + std::string plugin_name = "jdbc_drivers"; + std::string default_new = home + "/plugins/" + plugin_name; + std::string default_old = home + "/" + plugin_name; + std::string fname = "drv.jar"; + + touch_file(default_new, fname); + + std::string expected = "file://" + default_new + "/" + fname; + EXPECT_EQ(expected, path_util::check_and_return_default_plugin_url(fname, default_new, + plugin_name, home)); +} + +TEST(TestPathUtil, check_and_return_default_plugin_url_falls_back_to_old_default) { + std::string home = create_temp_home(); + std::string plugin_name = "jdbc_drivers"; + std::string default_new = home + "/plugins/" + plugin_name; + std::string default_old = home + "/" + plugin_name; + std::string fname = "drv.jar"; + + // create only old default file + touch_file(default_old, fname); + + std::string expected = "file://" + default_old + "/" + fname; + EXPECT_EQ(expected, path_util::check_and_return_default_plugin_url(fname, default_new, + plugin_name, home)); +} + +TEST(TestPathUtil, check_and_return_default_plugin_url_old_even_if_missing) { + std::string home = create_temp_home(); + std::string plugin_name = "jdbc_drivers"; + std::string default_new = home + "/plugins/" + plugin_name; + std::string default_old = home + "/" + plugin_name; + std::string fname = "drv.jar"; + + // neither new nor old has the file; should still point to old default + std::string expected = "file://" + default_old + "/" + fname; + EXPECT_EQ(expected, path_util::check_and_return_default_plugin_url(fname, default_new, + plugin_name, home)); +} + +TEST(TestPathUtil, check_and_return_default_plugin_url_custom_config_dir) { + std::string home = create_temp_home(); + std::string plugin_name = "jdbc_drivers"; + std::string custom_dir = home + "/custom/plugins"; + std::string fname = "drv.jar"; + touch_file(custom_dir, fname); + + std::string expected = "file://" + custom_dir + "/" + fname; + EXPECT_EQ(expected, + path_util::check_and_return_default_plugin_url(fname, custom_dir, plugin_name, home)); +} + +TEST(TestPathUtil, get_real_plugin_url_relative_paths) { + std::string home = create_temp_home(); + std::string plugin_name = "jdbc_drivers"; + std::string default_new = home + "/plugins/" + plugin_name; + std::string default_old = home + "/" + plugin_name; + std::string fname = "drv.jar"; + + // When new default exists + touch_file(default_new, fname); + std::string expected_new = "file://" + default_new + "/" + fname; + EXPECT_EQ(expected_new, path_util::get_real_plugin_url(fname, default_new, plugin_name, home)); + + // When only old default exists + std::string home2 = create_temp_home(); + std::string default_new2 = home2 + "/plugins/" + plugin_name; + std::string default_old2 = home2 + "/" + plugin_name; + touch_file(default_old2, fname); + std::string expected_old = "file://" + default_old2 + "/" + fname; + EXPECT_EQ(expected_old, + path_util::get_real_plugin_url(fname, default_new2, plugin_name, home2)); + + // When using a custom configured dir (not equal to default new path) + std::string custom_dir = home + "/custom"; + touch_file(custom_dir, fname); + std::string expected_custom = "file://" + custom_dir + "/" + fname; + EXPECT_EQ(expected_custom, + path_util::get_real_plugin_url(fname, custom_dir, plugin_name, home)); +} + } // namespace doris