From 0f3a0b6ed56dd52cc11d7a803f1c23c4973823b9 Mon Sep 17 00:00:00 2001 From: Aren Sandersen Date: Thu, 25 May 2023 14:22:47 -0700 Subject: [PATCH 1/4] Add rlm_python3 config parameter 'utf8_fail_as_bytes' which will return a byte string if it a string can't be interpreted as UTF8. --- src/modules/rlm_python3/rlm_python3.c | 46 +++++++++++++++++++-------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/src/modules/rlm_python3/rlm_python3.c b/src/modules/rlm_python3/rlm_python3.c index aaa43ab73d7d..743e9a493460 100644 --- a/src/modules/rlm_python3/rlm_python3.c +++ b/src/modules/rlm_python3/rlm_python3.c @@ -93,6 +93,7 @@ static CONF_PARSER module_config[] = { { "cext_compat", FR_CONF_OFFSET(PW_TYPE_BOOLEAN, rlm_python_t, cext_compat), "yes" }, { "pass_all_vps", FR_CONF_OFFSET(PW_TYPE_BOOLEAN, rlm_python_t, pass_all_vps), "no" }, { "pass_all_vps_dict", FR_CONF_OFFSET(PW_TYPE_BOOLEAN, rlm_python_t, pass_all_vps_dict), "no" }, + { "utf8_fail_as_bytes", FR_CONF_OFFSET(PW_TYPE_BOOLEAN, rlm_python_t, utf8_fail_as_bytes), "no" }, CONF_PARSER_TERMINATOR }; @@ -370,7 +371,7 @@ static void mod_vptuple(TALLOC_CTX *ctx, REQUEST *request, VALUE_PAIR **vps, PyO * Pass the value-pair print strings in a tuple. * */ -static int mod_populate_vptuple(PyObject *pPair, VALUE_PAIR *vp) +static int mod_populate_vptuple(PyObject *pPair, VALUE_PAIR *vp, utf8_fail_as_bytes) { PyObject *pStr = NULL; char buf[1024]; @@ -399,9 +400,28 @@ static int mod_populate_vptuple(PyObject *pPair, VALUE_PAIR *vp) pStr = PyUnicode_FromString(buf); if (pStr == NULL) { - ERROR("%s:%d, vp->da->name: %s", __func__, __LINE__, vp->da->name); if (PyErr_Occurred()) { - python_error_log(); + if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { + if (utf8_fail_as_bytes) { + DEBUG("Conversion to Unicode failed, returning %s as bytes", vp->da->name); + PyErr_Clear(); + pStr = PyBytes_FromString(buf); + if (pStr == NULL) { + ERROR("%s:%d, vp->da->name: %s", __func__, __LINE__, vp->da->name); + if (PyErr_Occurred()) { + python_error_log(); + } + return -1; + } + } else { + DEBUG("String is invalid utf8, use 'utf8_fail_as_bytes' config to return as bytes"); + return -1; + } + } else { + ERROR("%s:%d, vp->da->name: %s", __func__, __LINE__, vp->da->name); + python_error_log(); + return -1; + } } return -1; } @@ -465,7 +485,7 @@ static bool mod_populate_vps(PyObject* pArgs, const int pos, VALUE_PAIR *vps) return false; } -static rlm_rcode_t do_python_single(REQUEST *request, PyObject *pFunc, char const *funcname, bool pass_all_vps, bool pass_all_vps_dict) +static rlm_rcode_t do_python_single(REQUEST *request, PyObject *pFunc, char const *funcname, bool pass_all_vps, bool pass_all_vps_dict, bool utf8_fail_as_bytes) { PyObject *pRet = NULL; PyObject *pArgs = NULL; @@ -488,10 +508,10 @@ static rlm_rcode_t do_python_single(REQUEST *request, PyObject *pFunc, char cons /* If there is a request, fill in the first 4 attribute lists */ if (request != NULL) { - if (!mod_populate_vps(pArgs, 0, request->packet->vps) || - !mod_populate_vps(pArgs, 1, request->reply->vps) || - !mod_populate_vps(pArgs, 2, request->config) || - !mod_populate_vps(pArgs, 3, request->state)) { + if (!mod_populate_vps(pArgs, 0, request->packet->vps, utf8_fail_as_bytes) || + !mod_populate_vps(pArgs, 1, request->reply->vps, utf8_fail_as_bytes) || + !mod_populate_vps(pArgs, 2, request->config, utf8_fail_as_bytes) || + !mod_populate_vps(pArgs, 3, request->state, utf8_fail_as_bytes)) { ERROR("%s:%d, %s - mod_populate_vps failed", __func__, __LINE__, funcname); ret = RLM_MODULE_FAIL; @@ -501,7 +521,7 @@ static rlm_rcode_t do_python_single(REQUEST *request, PyObject *pFunc, char cons #ifdef WITH_PROXY /* fill proxy vps */ if (request->proxy) { - if (!mod_populate_vps(pArgs, 4, request->proxy->vps)) { + if (!mod_populate_vps(pArgs, 4, request->proxy->vps, utf8_fail_as_bytes)) { ERROR("%s:%d, %s - mod_populate_vps failed", __func__, __LINE__, funcname); ret = RLM_MODULE_FAIL; goto finish; @@ -509,13 +529,13 @@ static rlm_rcode_t do_python_single(REQUEST *request, PyObject *pFunc, char cons } else #endif { - mod_populate_vps(pArgs, 4, NULL); + mod_populate_vps(pArgs, 4, NULL, utf8_fail_as_bytes); } #ifdef WITH_PROXY /* fill proxy_reply vps */ if (request->proxy_reply) { - if (!mod_populate_vps(pArgs, 5, request->proxy_reply->vps)) { + if (!mod_populate_vps(pArgs, 5, request->proxy_reply->vps, utf8_fail_as_bytes)) { ERROR("%s:%d, %s - mod_populate_vps failed", __func__, __LINE__, funcname); ret = RLM_MODULE_FAIL; goto finish; @@ -523,12 +543,12 @@ static rlm_rcode_t do_python_single(REQUEST *request, PyObject *pFunc, char cons } else #endif { - mod_populate_vps(pArgs, 5, NULL); + mod_populate_vps(pArgs, 5, NULL, utf8_fail_as_bytes); } } /* If there is no request, set all the elements to None */ - else for (i = 0; i < 6; i++) mod_populate_vps(pArgs, i, NULL); + else for (i = 0; i < 6; i++) mod_populate_vps(pArgs, i, NULL, utf8_fail_as_bytes); /* * Call Python function. If pass_all_vps_dict is true, a dictionary with the From 5ceaeafb251b7cffaa2c57e6c8a3c1590c67946d Mon Sep 17 00:00:00 2001 From: "larry.sun" Date: Thu, 25 May 2023 22:29:50 -0700 Subject: [PATCH 2/4] Fixed sourcecode compilation errors --- raddb/mods-available/python3 | 4 ++++ src/modules/rlm_python3/rlm_python3.c | 12 ++++++------ src/modules/rlm_python3/rlm_python3.h | 1 + 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/raddb/mods-available/python3 b/raddb/mods-available/python3 index f0e0424c4e83..6166c1fc6b1d 100644 --- a/raddb/mods-available/python3 +++ b/raddb/mods-available/python3 @@ -27,6 +27,10 @@ python3 { # This option prevales over "pass_all_vps" # pass_all_vps_dict = no + # Added by Foxpass: + # Return a byte string if it a string can't be interpreted as UTF8 +# utf8_fail_as_bytes = no + # mod_instantiate = ${.module} # func_instantiate = instantiate diff --git a/src/modules/rlm_python3/rlm_python3.c b/src/modules/rlm_python3/rlm_python3.c index 743e9a493460..eaf392a8a5dc 100644 --- a/src/modules/rlm_python3/rlm_python3.c +++ b/src/modules/rlm_python3/rlm_python3.c @@ -371,7 +371,7 @@ static void mod_vptuple(TALLOC_CTX *ctx, REQUEST *request, VALUE_PAIR **vps, PyO * Pass the value-pair print strings in a tuple. * */ -static int mod_populate_vptuple(PyObject *pPair, VALUE_PAIR *vp, utf8_fail_as_bytes) +static int mod_populate_vptuple(PyObject *pPair, VALUE_PAIR *vp, bool utf8_fail_as_bytes) { PyObject *pStr = NULL; char buf[1024]; @@ -436,7 +436,7 @@ static int mod_populate_vptuple(PyObject *pPair, VALUE_PAIR *vp, utf8_fail_as_by * the indicated position in the tuple pArgs. * Returns false on error. */ -static bool mod_populate_vps(PyObject* pArgs, const int pos, VALUE_PAIR *vps) +static bool mod_populate_vps(PyObject* pArgs, const int pos, VALUE_PAIR *vps, bool utf8_fail_as_bytes) { PyObject *vps_tuple = NULL; int tuplelen = 0; @@ -469,7 +469,7 @@ static bool mod_populate_vps(PyObject* pArgs, const int pos, VALUE_PAIR *vps) /* The inside tuple has two only: */ if ((pPair = PyTuple_New(2)) == NULL) goto error; - if (mod_populate_vptuple(pPair, vp) == 0) { + if (mod_populate_vptuple(pPair, vp, utf8_fail_as_bytes) == 0) { /* Put the tuple inside the container */ PyTuple_SET_ITEM(vps_tuple, i, pPair); } else { @@ -824,7 +824,7 @@ static rlm_rcode_t do_python(rlm_python_t *inst, REQUEST *request, PyObject *pFu RDEBUG3("Using thread state %p", this_thread->state); PyEval_RestoreThread(this_thread->state); /* Swap in our local thread state */ - ret = do_python_single(request, pFunc, funcname, inst->pass_all_vps, inst->pass_all_vps_dict); + ret = do_python_single(request, pFunc, funcname, inst->pass_all_vps, inst->pass_all_vps_dict, inst->utf8_fail_as_bytes); PyEval_SaveThread(); return ret; @@ -1284,7 +1284,7 @@ static int mod_instantiate(CONF_SECTION *conf, void *instance) */ if (inst->instantiate.module_name && inst->instantiate.function_name) { - code = do_python_single(NULL, inst->instantiate.function, "instantiate", inst->pass_all_vps, inst->pass_all_vps_dict); + code = do_python_single(NULL, inst->instantiate.function, "instantiate", inst->pass_all_vps, inst->pass_all_vps_dict, inst->utf8_fail_as_bytes); if (code < 0) { error: python_error_log(); /* Needs valid thread with GIL */ @@ -1307,7 +1307,7 @@ static int mod_detach(void *instance) */ PyEval_RestoreThread(inst->sub_interpreter); - if (inst->detach.function) ret = do_python_single(NULL, inst->detach.function, "detach", inst->pass_all_vps, inst->pass_all_vps_dict); + if (inst->detach.function) ret = do_python_single(NULL, inst->detach.function, "detach", inst->pass_all_vps, inst->pass_all_vps_dict, inst->utf8_fail_as_bytes); #define PYTHON_FUNC_DESTROY(_x) python_function_destroy(&inst->_x) PYTHON_FUNC_DESTROY(instantiate); diff --git a/src/modules/rlm_python3/rlm_python3.h b/src/modules/rlm_python3/rlm_python3.h index 26cf29fa6d8d..bbca2a7f955a 100644 --- a/src/modules/rlm_python3/rlm_python3.h +++ b/src/modules/rlm_python3/rlm_python3.h @@ -46,6 +46,7 @@ typedef struct rlm_python_t { //!< made available to the python script. bool pass_all_vps; //!< Pass all VPS lists (request, reply, config, state, proxy_req, proxy_reply) bool pass_all_vps_dict; //!< Pass all VPS lists as a dictionary rather than a tuple + bool utf8_fail_as_bytes; //!< If not UTF8 string, convert it into bytes } rlm_python_t; /** Tracks a python module inst/thread state pair From 88ec159fa004c165642723acc386dad4c9de0258 Mon Sep 17 00:00:00 2001 From: "larry.sun" Date: Thu, 25 May 2023 22:55:34 -0700 Subject: [PATCH 3/4] fix Tab(4) replaced by 4xSpaces --- raddb/mods-available/python3 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/raddb/mods-available/python3 b/raddb/mods-available/python3 index 6166c1fc6b1d..98fbfc9afad2 100644 --- a/raddb/mods-available/python3 +++ b/raddb/mods-available/python3 @@ -27,9 +27,9 @@ python3 { # This option prevales over "pass_all_vps" # pass_all_vps_dict = no - # Added by Foxpass: - # Return a byte string if it a string can't be interpreted as UTF8 -# utf8_fail_as_bytes = no + # Added by Foxpass: + # Return a byte string if it a string can't be interpreted as UTF8 +# utf8_fail_as_bytes = no # mod_instantiate = ${.module} # func_instantiate = instantiate From 276393ebf9c02bfcaaff66f16c9ddec8c79a5f96 Mon Sep 17 00:00:00 2001 From: Larry Sun <3301779+larrysun@users.noreply.github.com> Date: Thu, 25 May 2023 23:39:44 -0700 Subject: [PATCH 4/4] update comments only --- raddb/mods-available/python3 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/raddb/mods-available/python3 b/raddb/mods-available/python3 index 98fbfc9afad2..b19d89d99686 100644 --- a/raddb/mods-available/python3 +++ b/raddb/mods-available/python3 @@ -27,8 +27,8 @@ python3 { # This option prevales over "pass_all_vps" # pass_all_vps_dict = no - # Added by Foxpass: - # Return a byte string if it a string can't be interpreted as UTF8 + # Return a byte string if the string has characters that can't be interpreted as UTF8. + # Otherwise, when invalid UTF8 is encountered, the module errors out. # utf8_fail_as_bytes = no # mod_instantiate = ${.module}