Skip to content

Commit

Permalink
Add rlm_python3 config parameter 'utf8_fail_as_bytes' which will retu…
Browse files Browse the repository at this point in the history 
…rn a byte string if it a string can't be interpreted as UTF8.
  • Loading branch information
@aren
aren committed Dec 2, 2023
1 parent 929c2bf commit d61b563
Show file tree
Hide file tree
Showing 3 changed files with 65 additions and 20 deletions.
4 changes: 4 additions & 0 deletions raddb/mods-available/python3
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ python3 {
# This option prevales over "pass_all_vps"
# pass_all_vps_dict = no

# Return a byte string if the string has characters that can't be interpreted as UTF8.
# Otherwise, when invalid UTF8 is encountered, the module errors out.
# utf8_fail_as_bytes = no

# mod_instantiate = ${.module}
# func_instantiate = instantiate

Expand Down
80 changes: 60 additions & 20 deletions src/modules/rlm_python3/rlm_python3.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ static CONF_PARSER module_config[] = {
{ "cext_compat", FR_CONF_OFFSET(PW_TYPE_BOOLEAN, rlm_python_t, cext_compat), "yes" },
{ "pass_all_vps", FR_CONF_OFFSET(PW_TYPE_BOOLEAN, rlm_python_t, pass_all_vps), "no" },
{ "pass_all_vps_dict", FR_CONF_OFFSET(PW_TYPE_BOOLEAN, rlm_python_t, pass_all_vps_dict), "no" },
{ "utf8_fail_as_bytes", FR_CONF_OFFSET(PW_TYPE_BOOLEAN, rlm_python_t, utf8_fail_as_bytes), "no" },

CONF_PARSER_TERMINATOR
};
Expand Down Expand Up @@ -370,7 +371,7 @@ static void mod_vptuple(TALLOC_CTX *ctx, REQUEST *request, VALUE_PAIR **vps, PyO
* Pass the value-pair print strings in a tuple.
*
*/
static int mod_populate_vptuple(PyObject *pPair, VALUE_PAIR *vp)
static int mod_populate_vptuple(PyObject *pPair, VALUE_PAIR *vp, bool utf8_fail_as_bytes)
{
PyObject *pStr = NULL;
char buf[1024];
Expand All @@ -387,6 +388,7 @@ static int mod_populate_vptuple(PyObject *pPair, VALUE_PAIR *vp)
ERROR("%s:%d, vp->da->name: %s", __func__, __LINE__, vp->da->name);
if (PyErr_Occurred()) {
python_error_log();
PyErr_Clear();
}

return -1;
Expand All @@ -399,9 +401,32 @@ static int mod_populate_vptuple(PyObject *pPair, VALUE_PAIR *vp)
pStr = PyUnicode_FromString(buf);

if (pStr == NULL) {
ERROR("%s:%d, vp->da->name: %s", __func__, __LINE__, vp->da->name);
if (PyErr_Occurred()) {
python_error_log();
if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
PyErr_Clear();
if (utf8_fail_as_bytes) {
DEBUG("Conversion to Unicode failed, returning %s as bytes", vp->da->name);
pStr = PyBytes_FromString(buf);
if (pStr == NULL) {
ERROR("%s:%d, vp->da->name: %s", __func__, __LINE__, vp->da->name);
/* Did we get another exception while handling the UnicodeDecodeError? */
if (PyErr_Occurred()) {
python_error_log();
PyErr_Clear();
}
return -1;
}
} else {
ERROR("String is invalid utf8, use 'utf8_fail_as_bytes' config to return as bytes");
return -1;
}
/* Exception was not UnicodeDecodeError */
} else {
ERROR("%s:%d, vp->da->name: %s", __func__, __LINE__, vp->da->name);
python_error_log();
PyErr_Clear();
return -1;
}
}
return -1;
}
Expand All @@ -416,7 +441,7 @@ static int mod_populate_vptuple(PyObject *pPair, VALUE_PAIR *vp)
* the indicated position in the tuple pArgs.
* Returns false on error.
*/
static bool mod_populate_vps(PyObject* pArgs, const int pos, VALUE_PAIR *vps)
static bool mod_populate_vps(PyObject* pArgs, const int pos, VALUE_PAIR *vps, bool utf8_fail_as_bytes)
{
PyObject *vps_tuple = NULL;
int tuplelen = 0;
Expand All @@ -441,18 +466,25 @@ static bool mod_populate_vps(PyObject* pArgs, const int pos, VALUE_PAIR *vps)
for (vp = fr_cursor_init(&cursor, &vps); vp; vp = fr_cursor_next(&cursor))
tuplelen++;

if ((vps_tuple = PyTuple_New(tuplelen)) == NULL) goto error;
if ((vps_tuple = PyTuple_New(tuplelen)) == NULL) {
ERROR("%s:%d - Memory cannot be allocated for PyTyple_New", __func__, __LINE__);
goto error;
}

for (vp = fr_cursor_init(&cursor, &vps); vp; vp = fr_cursor_next(&cursor), i++) {
PyObject *pPair = NULL;

/* The inside tuple has two only: */
if ((pPair = PyTuple_New(2)) == NULL) goto error;
if ((pPair = PyTuple_New(2)) == NULL) {
ERROR("%s:%d - Memory cannot be allocated for PyTyple_New", __func__, __LINE__);
goto error;
}

if (mod_populate_vptuple(pPair, vp) == 0) {
if (mod_populate_vptuple(pPair, vp, utf8_fail_as_bytes) == 0) {
/* Put the tuple inside the container */
PyTuple_SET_ITEM(vps_tuple, i, pPair);
} else {
ERROR("%s:%d - mod_populate_vptuple failed", __func__, __LINE__);
Py_DECREF(pPair);
goto error;
}
Expand All @@ -465,7 +497,7 @@ static bool mod_populate_vps(PyObject* pArgs, const int pos, VALUE_PAIR *vps)
return false;
}

static rlm_rcode_t do_python_single(REQUEST *request, PyObject *pFunc, char const *funcname, bool pass_all_vps, bool pass_all_vps_dict)
static rlm_rcode_t do_python_single(REQUEST *request, PyObject *pFunc, char const *funcname, bool pass_all_vps, bool pass_all_vps_dict, bool utf8_fail_as_bytes)
{
PyObject *pRet = NULL;
PyObject *pArgs = NULL;
Expand All @@ -488,10 +520,10 @@ static rlm_rcode_t do_python_single(REQUEST *request, PyObject *pFunc, char cons

/* If there is a request, fill in the first 4 attribute lists */
if (request != NULL) {
if (!mod_populate_vps(pArgs, 0, request->packet->vps) ||
!mod_populate_vps(pArgs, 1, request->reply->vps) ||
!mod_populate_vps(pArgs, 2, request->config) ||
!mod_populate_vps(pArgs, 3, request->state)) {
if (!mod_populate_vps(pArgs, 0, request->packet->vps, utf8_fail_as_bytes) ||
!mod_populate_vps(pArgs, 1, request->reply->vps, utf8_fail_as_bytes) ||
!mod_populate_vps(pArgs, 2, request->config, utf8_fail_as_bytes) ||
!mod_populate_vps(pArgs, 3, request->state, utf8_fail_as_bytes)) {

ERROR("%s:%d, %s - mod_populate_vps failed", __func__, __LINE__, funcname);
ret = RLM_MODULE_FAIL;
Expand All @@ -501,34 +533,34 @@ static rlm_rcode_t do_python_single(REQUEST *request, PyObject *pFunc, char cons
#ifdef WITH_PROXY
/* fill proxy vps */
if (request->proxy) {
if (!mod_populate_vps(pArgs, 4, request->proxy->vps)) {
if (!mod_populate_vps(pArgs, 4, request->proxy->vps, utf8_fail_as_bytes)) {
ERROR("%s:%d, %s - mod_populate_vps failed", __func__, __LINE__, funcname);
ret = RLM_MODULE_FAIL;
goto finish;
}
} else
#endif
{
mod_populate_vps(pArgs, 4, NULL);
mod_populate_vps(pArgs, 4, NULL, utf8_fail_as_bytes);
}

#ifdef WITH_PROXY
/* fill proxy_reply vps */
if (request->proxy_reply) {
if (!mod_populate_vps(pArgs, 5, request->proxy_reply->vps)) {
if (!mod_populate_vps(pArgs, 5, request->proxy_reply->vps, utf8_fail_as_bytes)) {
ERROR("%s:%d, %s - mod_populate_vps failed", __func__, __LINE__, funcname);
ret = RLM_MODULE_FAIL;
goto finish;
}
} else
#endif
{
mod_populate_vps(pArgs, 5, NULL);
mod_populate_vps(pArgs, 5, NULL, utf8_fail_as_bytes);
}

}
/* If there is no request, set all the elements to None */
else for (i = 0; i < 6; i++) mod_populate_vps(pArgs, i, NULL);
else for (i = 0; i < 6; i++) mod_populate_vps(pArgs, i, NULL, utf8_fail_as_bytes);

/*
* Call Python function. If pass_all_vps_dict is true, a dictionary with the
Expand Down Expand Up @@ -567,6 +599,7 @@ static rlm_rcode_t do_python_single(REQUEST *request, PyObject *pFunc, char cons
ERROR("%s:%d, %s - pRet is NULL", __func__, __LINE__, funcname);
if (PyErr_Occurred()) {
python_error_log();
PyErr_Clear();
}
ret = RLM_MODULE_FAIL;
goto finish;
Expand Down Expand Up @@ -674,6 +707,13 @@ static rlm_rcode_t do_python_single(REQUEST *request, PyObject *pFunc, char cons
if (ret == RLM_MODULE_FAIL) {
ERROR("%s:%d, %s - RLM_MODULE_FAIL", __func__, __LINE__, funcname);
}

if (PyErr_Occurred()) {
ERROR("Unhandled Python exception (see below); clearing.");
python_error_log();
PyErr_Clear();
}

return ret;
}

Expand Down Expand Up @@ -804,7 +844,7 @@ static rlm_rcode_t do_python(rlm_python_t *inst, REQUEST *request, PyObject *pFu
RDEBUG3("Using thread state %p", this_thread->state);

PyEval_RestoreThread(this_thread->state); /* Swap in our local thread state */
ret = do_python_single(request, pFunc, funcname, inst->pass_all_vps, inst->pass_all_vps_dict);
ret = do_python_single(request, pFunc, funcname, inst->pass_all_vps, inst->pass_all_vps_dict, inst->utf8_fail_as_bytes);
PyEval_SaveThread();

return ret;
Expand Down Expand Up @@ -1264,7 +1304,7 @@ static int mod_instantiate(CONF_SECTION *conf, void *instance)
*/
if (inst->instantiate.module_name && inst->instantiate.function_name) {

code = do_python_single(NULL, inst->instantiate.function, "instantiate", inst->pass_all_vps, inst->pass_all_vps_dict);
code = do_python_single(NULL, inst->instantiate.function, "instantiate", inst->pass_all_vps, inst->pass_all_vps_dict, inst->utf8_fail_as_bytes);
if (code < 0) {
error:
python_error_log(); /* Needs valid thread with GIL */
Expand All @@ -1287,7 +1327,7 @@ static int mod_detach(void *instance)
*/
PyEval_RestoreThread(inst->sub_interpreter);

if (inst->detach.function) ret = do_python_single(NULL, inst->detach.function, "detach", inst->pass_all_vps, inst->pass_all_vps_dict);
if (inst->detach.function) ret = do_python_single(NULL, inst->detach.function, "detach", inst->pass_all_vps, inst->pass_all_vps_dict, inst->utf8_fail_as_bytes);

#define PYTHON_FUNC_DESTROY(_x) python_function_destroy(&inst->_x)
PYTHON_FUNC_DESTROY(instantiate);
Expand Down
1 change: 1 addition & 0 deletions src/modules/rlm_python3/rlm_python3.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ typedef struct rlm_python_t {
//!< made available to the python script.
bool pass_all_vps; //!< Pass all VPS lists (request, reply, config, state, proxy_req, proxy_reply)
bool pass_all_vps_dict; //!< Pass all VPS lists as a dictionary rather than a tuple
bool utf8_fail_as_bytes; //!< If not UTF8 string, convert it into bytes
} rlm_python_t;

/** Tracks a python module inst/thread state pair
Expand Down

0 comments on commit d61b563

Please sign in to comment.