From b4a9031cdbb5503b61aea611af4b3196524c36b6 Mon Sep 17 00:00:00 2001 From: Philipp Serr Date: Fri, 27 Jan 2017 18:16:30 +0100 Subject: [PATCH 1/4] Add module_restart_next definition to swig file --- pythonmod/interface.i | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pythonmod/interface.i b/pythonmod/interface.i index 5f2559bac..c76d017aa 100644 --- a/pythonmod/interface.i +++ b/pythonmod/interface.i @@ -1128,6 +1128,7 @@ struct delegpt { %rename ("MODULE_STATE_INITIAL") "module_state_initial"; %rename ("MODULE_WAIT_REPLY") "module_wait_reply"; %rename ("MODULE_WAIT_MODULE") "module_wait_module"; +%rename ("MODULE_RESTART_NEXT") "module_restart_next"; %rename ("MODULE_WAIT_SUBQUERY") "module_wait_subquery"; %rename ("MODULE_ERROR") "module_error"; %rename ("MODULE_FINISHED") "module_finished"; @@ -1136,6 +1137,7 @@ enum module_ext_state { module_state_initial = 0, module_wait_reply, module_wait_module, + module_restart_next, module_wait_subquery, module_error, module_finished From 7cc4ff05ad7e4a324be5a9fd948ae5230f832934 Mon Sep 17 00:00:00 2001 From: Philipp Serr Date: Sat, 21 Jan 2017 15:47:54 +0100 Subject: [PATCH 2/4] Initialize per-query qdata and per-module mod_env The python module used to assign None to the per-query (qdata argument) and per-module (mod_env variable) data stores. Hence, there was no obvious way for python code to use these data stores. This commit initializes both data stores with a dict instance. --- pythonmod/pythonmod.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pythonmod/pythonmod.c b/pythonmod/pythonmod.c index a668ecc23..02908af56 100644 --- a/pythonmod/pythonmod.c +++ b/pythonmod/pythonmod.c @@ -317,8 +317,7 @@ int pythonmod_init(struct module_env* env, int id) /* Load file */ pe->module = PyImport_AddModule("__main__"); pe->dict = PyModule_GetDict(pe->module); - pe->data = Py_None; - Py_INCREF(pe->data); + pe->data = PyDict_New(); PyModule_AddObject(pe->module, "mod_env", pe->data); /* TODO: deallocation of pe->... if an error occurs */ @@ -485,8 +484,7 @@ void pythonmod_operate(struct module_qstate* qstate, enum module_ev event, pq = qstate->minfo[id] = malloc(sizeof(struct pythonmod_qstate)); /* Initialize per query data */ - pq->data = Py_None; - Py_INCREF(pq->data); + pq->data = PyDict_New(); } /* Call operate */ From b248654aabe5e03c9502ad2c8991514cbcd897d3 Mon Sep 17 00:00:00 2001 From: Philipp Serr Date: Sun, 22 Jan 2017 11:58:28 +0100 Subject: [PATCH 3/4] Support multiple python module instances This commit adds proper support for multiple instances of the python module: When more than one instance is added to the module list, the first instance loads the first script specified in the `python:` configuration section. The second instance loads the second script, and so on. When there are more module instances in the module list than there are scripts in the `python:` section, an error is raised during initialization and unbound won't start. When more scripts than module instances are provided, the surplus scripts are ignored. --- pythonmod/interface.i | 2 +- pythonmod/pythonmod.c | 79 ++++++++++++++++++++++++++++--------------- util/config_file.c | 30 ++++++++++++++-- util/config_file.h | 10 +++++- util/configparser.c | 4 +-- util/configparser.y | 4 +-- 6 files changed, 93 insertions(+), 36 deletions(-) diff --git a/pythonmod/interface.i b/pythonmod/interface.i index 5f2559bac..465a9184d 100644 --- a/pythonmod/interface.i +++ b/pythonmod/interface.i @@ -1026,7 +1026,7 @@ struct config_file { char* control_key_file; char* control_cert_file; int do_daemonize; - char* python_script; + struct config_strlist* python_script; }; /* ************************************************************************************ * diff --git a/pythonmod/pythonmod.c b/pythonmod/pythonmod.c index a668ecc23..a098c326e 100644 --- a/pythonmod/pythonmod.c +++ b/pythonmod/pythonmod.c @@ -64,6 +64,15 @@ typedef struct PyThreadState PyThreadState; typedef void* PyGILState_STATE; #endif +/** + * counter for python module instances + * incremented by pythonmod_init(...) + */ +int py_mod_count = 0; + +/** Python main thread */ +PyThreadState* mainthr; + /** * Global state for the module. */ @@ -72,8 +81,6 @@ struct pythonmod_env { /** Python script filename. */ const char* fname; - /** Python main thread */ - PyThreadState* mainthr; /** Python module. */ PyObject* module; @@ -242,11 +249,15 @@ log_py_err(void) int pythonmod_init(struct module_env* env, int id) { + int py_mod_idx = py_mod_count++; + /* Initialize module */ FILE* script_py = NULL; PyObject* py_init_arg, *res; PyGILState_STATE gil; - int init_standard = 1; + int init_standard = 1, i = 0; + + struct config_strlist* cfg_item = env->cfg->python_script; struct pythonmod_env* pe = (struct pythonmod_env*)calloc(1, sizeof(struct pythonmod_env)); if (!pe) @@ -258,14 +269,21 @@ int pythonmod_init(struct module_env* env, int id) env->modinfo[id] = (void*) pe; /* Initialize module */ - pe->fname = env->cfg->python_script; + pe->fname=NULL; i = 0; + while (cfg_item!=NULL) { + if (py_mod_idx==i++) { + pe->fname=cfg_item->str; + break; + } + cfg_item = cfg_item->next; + } if(pe->fname==NULL || pe->fname[0]==0) { - log_err("pythonmod: no script given."); + log_err("pythonmod[%d]: no script given.", py_mod_idx); return 0; } /* Initialize Python libraries */ - if (!Py_IsInitialized()) + if (py_mod_count==1 && !Py_IsInitialized()) { #if PY_MAJOR_VERSION >= 3 wchar_t progname[8]; @@ -281,29 +299,31 @@ int pythonmod_init(struct module_env* env, int id) Py_Initialize(); PyEval_InitThreads(); SWIG_init(); - pe->mainthr = PyEval_SaveThread(); + mainthr = PyEval_SaveThread(); } gil = PyGILState_Ensure(); - /* Initialize Python */ - PyRun_SimpleString("import sys \n"); - PyRun_SimpleString("sys.path.append('.') \n"); - if(env->cfg->directory && env->cfg->directory[0]) { - char wdir[1524]; - snprintf(wdir, sizeof(wdir), "sys.path.append('%s') \n", - env->cfg->directory); - PyRun_SimpleString(wdir); - } - PyRun_SimpleString("sys.path.append('"RUN_DIR"') \n"); - PyRun_SimpleString("sys.path.append('"SHARE_DIR"') \n"); - PyRun_SimpleString("import distutils.sysconfig \n"); - PyRun_SimpleString("sys.path.append(distutils.sysconfig.get_python_lib(1,0)) \n"); - if (PyRun_SimpleString("from unboundmodule import *\n") < 0) - { - log_err("pythonmod: cannot initialize core module: unboundmodule.py"); - PyGILState_Release(gil); - return 0; + if (py_mod_count==1) { + /* Initialize Python */ + PyRun_SimpleString("import sys \n"); + PyRun_SimpleString("sys.path.append('.') \n"); + if(env->cfg->directory && env->cfg->directory[0]) { + char wdir[1524]; + snprintf(wdir, sizeof(wdir), "sys.path.append('%s') \n", + env->cfg->directory); + PyRun_SimpleString(wdir); + } + PyRun_SimpleString("sys.path.append('"RUN_DIR"') \n"); + PyRun_SimpleString("sys.path.append('"SHARE_DIR"') \n"); + PyRun_SimpleString("import distutils.sysconfig \n"); + PyRun_SimpleString("sys.path.append(distutils.sysconfig.get_python_lib(1,0)) \n"); + if (PyRun_SimpleString("from unboundmodule import *\n") < 0) + { + log_err("pythonmod: cannot initialize core module: unboundmodule.py"); + PyGILState_Release(gil); + return 0; + } } /* Check Python file load */ @@ -341,6 +361,7 @@ int pythonmod_init(struct module_env* env, int id) (void)PyParser_SimpleParseFile(script_py, pe->fname, Py_file_input); log_py_err(); PyGILState_Release(gil); + fclose(script_py); return 0; } fclose(script_py); @@ -425,9 +446,11 @@ void pythonmod_deinit(struct module_env* env, int id) Py_XDECREF(pe->data); PyGILState_Release(gil); - PyEval_RestoreThread(pe->mainthr); - Py_Finalize(); - pe->mainthr = NULL; + if(--py_mod_count==0) { + PyEval_RestoreThread(mainthr); + Py_Finalize(); + mainthr = NULL; + } } pe->fname = NULL; free(pe); diff --git a/util/config_file.c b/util/config_file.c index 9b60254d7..8d6bdf5ea 100644 --- a/util/config_file.c +++ b/util/config_file.c @@ -602,7 +602,7 @@ int config_set_option(struct config_file* cfg, const char* opt, else S_STR("control-key-file:", control_key_file) else S_STR("control-cert-file:", control_cert_file) else S_STR("module-config:", module_conf) - else S_STR("python-script:", python_script) + else S_STRLIST("python-script:", python_script) else S_YNO("disable-dnssec-lame-check:", disable_dnssec_lame_check) #ifdef CLIENT_SUBNET /* Can't set max subnet prefix here, since that value is used when @@ -1054,7 +1054,7 @@ config_get_option(struct config_file* cfg, const char* opt, else O_YNO(opt, "unblock-lan-zones", unblock_lan_zones) else O_YNO(opt, "insecure-lan-zones", insecure_lan_zones) else O_DEC(opt, "max-udp-size", max_udp_size) - else O_STR(opt, "python-script", python_script) + else O_LST(opt, "python-script", python_script) else O_YNO(opt, "disable-dnssec-lame-check", disable_dnssec_lame_check) else O_DEC(opt, "ip-ratelimit", ip_ratelimit) else O_DEC(opt, "ratelimit", ratelimit) @@ -1420,6 +1420,7 @@ config_delete(struct config_file* cfg) free(cfg->dnstap_version); config_deldblstrlist(cfg->ratelimit_for_domain); config_deldblstrlist(cfg->ratelimit_below_domain); + config_delstrlist(cfg->python_script); #ifdef USE_IPSECMOD free(cfg->ipsecmod_hook); config_delstrlist(cfg->ipsecmod_whitelist); @@ -1630,6 +1631,31 @@ cfg_strlist_insert(struct config_strlist** head, char* item) return 1; } +int +cfg_strlist_append_ex(struct config_strlist** head, char* item) +{ + struct config_strlist *s; + if(!item || !head) + return 0; + s = (struct config_strlist*)calloc(1, sizeof(struct config_strlist)); + if(!s) + return 0; + s->str = item; + s->next = NULL; + + if (*head==NULL) { + *head = s; + } else { + struct config_strlist *last = *head; + while (last->next!=NULL) { + last = last->next; + } + last->next = s; + } + + return 1; +} + int cfg_str2list_insert(struct config_str2list** head, char* item, char* i2) { diff --git a/util/config_file.h b/util/config_file.h index 3cffdbff9..68d1c911c 100644 --- a/util/config_file.h +++ b/util/config_file.h @@ -433,7 +433,7 @@ struct config_file { char* control_cert_file; /** Python script file */ - char* python_script; + struct config_strlist* python_script; /** Use systemd socket activation. */ int use_systemd; @@ -820,6 +820,14 @@ char* config_collate_cat(struct config_strlist* list); */ int cfg_strlist_append(struct config_strlist_head* list, char* item); +/** + * Searches the end of a string list and appends the given text. + * @param head: pointer to strlist head variable. + * @param item: new item. malloced by caller. if NULL the insertion fails. + * @return true on success. + */ +int cfg_strlist_append_ex(struct config_strlist** head, char* item); + /** * Find string in strlist. * @param head: pointer to strlist head variable. diff --git a/util/configparser.c b/util/configparser.c index 9674aa283..2b5ed3f68 100644 --- a/util/configparser.c +++ b/util/configparser.c @@ -5716,8 +5716,8 @@ yyparse (void) #line 2721 "./util/configparser.y" /* yacc.c:1648 */ { OUTYY(("P(python-script:%s)\n", (yyvsp[0].str))); - free(cfg_parser->cfg->python_script); - cfg_parser->cfg->python_script = (yyvsp[0].str); + if(!cfg_strlist_append_ex(&cfg_parser->cfg->python_script, (yyvsp[0].str))) + yyerror("out of memory"); } #line 5723 "util/configparser.c" /* yacc.c:1648 */ break; diff --git a/util/configparser.y b/util/configparser.y index 5f52f4d77..9ca2c35ec 100644 --- a/util/configparser.y +++ b/util/configparser.y @@ -2720,8 +2720,8 @@ content_py: py_script py_script: VAR_PYTHON_SCRIPT STRING_ARG { OUTYY(("P(python-script:%s)\n", $2)); - free(cfg_parser->cfg->python_script); - cfg_parser->cfg->python_script = $2; + if(!cfg_strlist_append_ex(&cfg_parser->cfg->python_script, $2)) + yyerror("out of memory"); } server_disable_dnssec_lame_check: VAR_DISABLE_DNSSEC_LAME_CHECK STRING_ARG { From bfae29866bed4e0162432758875a5ac72ffd535f Mon Sep 17 00:00:00 2001 From: Philipp Serr Date: Thu, 26 Jan 2017 21:24:29 +0100 Subject: [PATCH 4/4] Document how to configure multiple python modules --- doc/unbound.conf.5.in | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/unbound.conf.5.in b/doc/unbound.conf.5.in index cbb7b654b..d8a2a3513 100644 --- a/doc/unbound.conf.5.in +++ b/doc/unbound.conf.5.in @@ -1752,7 +1752,8 @@ clause gives the settings for the \fIpython\fR(1) script module. This module acts like the iterator and validator modules do, on queries and answers. To enable the script module it has to be compiled into the daemon, and the word "python" has to be put in the \fBmodule\-config:\fR option -(usually first, or between the validator and iterator). +(usually first, or between the validator and iterator). Multiple instances of +the python module are supported by adding the word "python" more than once. .LP If the \fBchroot:\fR option is enabled, you should make sure Python's library directory structure is bind mounted in the new root environment, see @@ -1761,7 +1762,8 @@ absolute path relative to the new root, or as a relative path to the working directory. .TP .B python\-script: \fI\fR -The script file to load. +The script file to load. Repeat this option for every python module instance +added to the \fBmodule\-config:\fR option. .SS "DNS64 Module Options" .LP The dns64 module must be configured in the \fBmodule\-config:\fR "dns64