diff --git a/models/tasks.py b/models/tasks.py old mode 100755 new mode 100644 index 101ea6b5e1..532748b08a --- a/models/tasks.py +++ b/models/tasks.py @@ -60,17 +60,17 @@ def process_inbound_email(username): tasks["process_inbound_email"] = process_inbound_email - # ------------------------------------------------------------------------- - def process_log(): + # ----------------------------------------------------------------------------- + def parse_workflow(workflow): """ - Processes the msg_log for unparsed messages. + Processes the msg_log for unparsed messages. """ # Run the Task - result = msg.process_log() + result = msg.parse_import(workflow) return result + + tasks["parse_workflow"] = parse_workflow - tasks["process_log"] = process_log - # ----------------------------------------------------------------------------- def sync_synchronize(repository_id, user_id=None, manual=False): """ diff --git a/models/zzz_1st_run.py b/models/zzz_1st_run.py index 8d2124f5a7..b3e2dd01bc 100644 --- a/models/zzz_1st_run.py +++ b/models/zzz_1st_run.py @@ -22,6 +22,20 @@ # Scheduled Tasks if deployment_settings.has_module("msg"): + + #Message Parsing Tasks for each enabled workflows + #for workflow in deployment_settings.get_parser_enabled(): + #workflow = int(workflow.split("_")[1]) + #s3task.schedule_task("parse_workflow", + #vars={"workflow": workflow}, + #period=300, # seconds + #timeout=300, # seconds + #repeats=0 # unlimited + #) + + + + # Send Messages from Outbox # SMS every minute s3task.schedule_task("process_outbox", @@ -49,13 +63,7 @@ # repeats=0 # unlimited # ) - #Process the msg_log for unparsed messages. - #s3task.schedule_task("process_log", - # period=300, # seconds - # timeout=300, # seconds - # repeats=0 # unlimited - # ) - + # Person Registry tablename = "pr_person" table = db[tablename] diff --git a/modules/eden/msg.py b/modules/eden/msg.py index f8e2706b12..60a1d18d77 100644 --- a/modules/eden/msg.py +++ b/modules/eden/msg.py @@ -35,6 +35,7 @@ "S3TropoModel", "S3TwitterModel", "S3XFormsModel", + "S3ParsingModel" ] from gluon import * @@ -71,6 +72,10 @@ def model(self): 2:T("Medium"), 1:T("Low") } + source_task_id = S3ReusableField("source_task_id", db.scheduler_task, + requires = IS_NULL_OR(IS_ONE_OF(db, "scheduler_task.id")), + ondelete = "RESTRICT") + # --------------------------------------------------------------------- # Message Log - all Inbound & Outbound Messages # --------------------------------------------------------------------- @@ -100,7 +105,8 @@ def model(self): (status and ["Parsed"] or ["Not Parsed"])[0], label = T("Parsing Status")), Field("reply", "text" , - label = T("Reply")), + label = T("Reply")), + source_task_id(label="Source ID"), *s3.meta_fields()) self.configure(tablename, @@ -118,7 +124,8 @@ def model(self): #"actioned_comments", #"priority", "is_parsed", - "reply" + "reply", + "source_task_id" ]) # Components @@ -716,5 +723,40 @@ def model(self): # --------------------------------------------------------------------- return Storage() + +# --------------------------------------------------------------------- + +class S3ParsingModel(S3Model): + """ + Message Parsing Model + """ + + names = ["msg_workflow"] + + def model(self): + + T = current.T + s3 = current.response.s3 + db = current.db + # Reusable Source Task ID + source_task_id = S3ReusableField("source_task_id", db.scheduler_task, + requires = IS_NULL_OR(IS_ONE_OF(db, "scheduler_task.id")), + ondelete = "RESTRICT") + # Reusable Workflow Task ID + workflow_task_id = S3ReusableField("workflow_task_id", db.scheduler_task, + requires = IS_NULL_OR(IS_ONE_OF(db, "scheduler_task.id")), + ondelete = "RESTRICT") + + tablename = "msg_workflow" + table = self.define_table(tablename, + source_task_id(), + workflow_task_id(), + *s3.meta_fields()) + + + return Storage() + + + # END ========================================================================= diff --git a/modules/s3/s3msg.py b/modules/s3/s3msg.py old mode 100755 new mode 100644 index 9ba69b4e97..cf6ed3a7f2 --- a/modules/s3/s3msg.py +++ b/modules/s3/s3msg.py @@ -188,192 +188,39 @@ def receive_msg(subject="", # Explicitly commit DB operations when running from Cron db.commit() return True - - # ------------------------------------------------------------------------- + + # ----------------------------------------------------------------------------- # Parser for inbound messages # ----------------------------------------------------------------------------- @staticmethod - def parse_message(message=""): + def parse_import(workflow): """ - Parse Incoming Message according to keyword - - @ToDo: Check for OpenGeoSMS - - route SI to IRS - - @ToDo: Allow this to be more easily customised by moving the - routing logic to a separate file (ideally web configurable) + Parsing Workflow Importer. """ - - if not message: - return None - - T = current.T + + from parser import S3Parsing db = current.db - s3db = current.s3db - s3mgr = current.manager - - primary_keywords = ["get", "give", "show"] # Equivalent keywords in one list - contact_keywords = ["email", "mobile", "facility", "clinical", - "security", "phone", "status", "hospital", - "person", "organisation"] - - pkeywords = primary_keywords+contact_keywords - keywords = string.split(message) - pquery = [] - name = "" + ltable = current.s3db.msg_log + wtable = current.s3db.msg_workflow + + query = (wtable.workflow_task_id == workflow) + records = db(query).select(wtable.source_task_id) reply = "" - for word in keywords: - match = None - for key in pkeywords: - if soundex(key) == soundex(word): - match = key - break - if match: - pquery.append(match) - else: - name = word - - # --------------------------------------------------------------------- - # Person Search [get name person phone email] - if "person" in pquery: - - table = s3db.pr_person - rows = db(table.id > 0).select(table.pe_id, - table.first_name, - table.middle_name, - table.last_name) - for row in rows: - result = [] - if (soundex(str(name)) == soundex(str(row.first_name))) or \ - (soundex(str(name)) == soundex(str(row.middle_name))) or \ - (soundex(str(name)) == soundex(str(row.last_name))): - presult = dict(name = row.first_name, id = row.pe_id) - result.append(presult) - break - - if len(result) > 1: - return T("Multiple Matches") - if len(result) == 1: - reply = result[0]["name"] - table = s3db.pr_contact - if "email" in pquery: - query = (table.pe_id == result[0]["id"]) & \ - (table.contact_method == "EMAIL") - recipient = db(query).select(table.value, - orderby = table.priority, - limitby=(0, 1)).first() - reply = "%s Email->%s" % (reply, recipient.value) - if "phone" in pquery: - query = (table.pe_id == result[0]["id"]) & \ - (table.contact_method == "SMS") - recipient = db(query).select(table.value, - orderby = table.priority, - limitby=(0, 1)).first() - reply = "%s Mobile->%s" % (reply, - recipient.value) - - if len(result) == 0: - return T("No Match") - - return reply - - # --------------------------------------------------------------------- - # Hospital Search [example: get name hospital facility status ] - if "hospital" in pquery: - table = s3db.hms_hospital - rows = db(table.id > 0).select(table.id, - table.name, - table.aka1, - table.aka2) - for row in rows: - result = [] - if (soundex(str(name)) == soundex(str(row.name))) or \ - (soundex(name) == soundex(str(row.aka1))) or \ - (soundex(name) == soundex(str(row.aka2))): - result.append(row) - break - - - if len(result) > 1: - return T("Multiple Matches") - - if len(result) == 1: - hospital = db(table.id == result[0].id).select().first() - reply = "%s %s (%s) " % (reply, hospital.name, - T("Hospital")) - if "phone" in pquery: - reply = reply + "Phone->" + str(hospital.phone_emergency) - if "facility" in pquery: - reply = reply + "Facility status " + str(table.facility_status.represent(hospital.facility_status)) - if "clinical" in pquery: - reply = reply + "Clinical status " + str(table.clinical_status.represent(hospital.clinical_status)) - if "security" in pquery: - reply = reply + "Security status " + str(table.security_status.represent(hospital.security_status)) - - if len(result) == 0: - return T("No Match") - - return reply - - # --------------------------------------------------------------------- - # Organization search [example: get name organisation phone] - if "organisation" in pquery: - table = s3db.org_organisation - rows = db(table.id > 0).select(table.id, - table.name, - table.acronym) + for record in records: + query = (ltable.is_parsed == False) & \ + (ltable.inbound == True) &\ + (ltable.source_task_id == record.source_task_id) + rows = db(query).select() + for row in rows: - result = [] - if (soundex(str(name)) == soundex(str(row.name))) or \ - (soundex(str(name)) == soundex(str(row.acronym))): - result.append(row) - break - - if len(result) > 1: - return T("Multiple Matches") - - if len(result) == 1: - organisation = db(table.id == result[0].id).select().first() - reply = "%s %s (%s) " % (reply, organisation.name, - T("Organization")) - if "phone" in pquery: - reply = reply + "Phone->" + str(organisation.donation_phone) - if "office" in pquery: - reply = reply + "Address->" + s3_get_db_field_value(tablename = "org_office", - fieldname = "address", - look_up_value = organisation.id) - if len(reply) == 0: - return T("No Match") - - return reply - - return "Please provide one of the keywords - person, hospital, organisation" - - - - # ========================================================================= - # Processing of Unparsed Messages - # ========================================================================= - def process_log(self): - """ - Processes the unparsed messages in msg_log - """ - - db = current.db - ltable = current.s3db.msg_log - - query = (ltable.is_parsed == False) & \ - (ltable.inbound == True) - rows = db(query).select() - - for row in rows: - message = row.message - reply = self.parse_message(message) - db(ltable.id == row.id).update(reply = reply,is_parsed = True) - - return - - + message = row .message + reply = S3Parsing.parser(workflow, message) + db(ltable.id == row.id).update(reply = reply,is_parsed = True) + db.commit() + + return + + # ========================================================================= # Outbound Messages # ========================================================================= diff --git a/modules/s3cfg.py b/modules/s3cfg.py index be654b2717..1614107161 100644 --- a/modules/s3cfg.py +++ b/modules/s3cfg.py @@ -65,7 +65,7 @@ def __init__(self): self.hrm = Storage() self.project = Storage() self.save_search = Storage() - + self.parser = Storage() # ------------------------------------------------------------------------- # Template def get_template(self): @@ -762,7 +762,12 @@ def get_save_search_widget(self): Enable the Saved Search widget """ return self.save_search.get("widget", True) - + # ------------------------------------------------------------------------- + # Message Parser Settings + def get_parser_enabled(self): + return self.parser.get("parser_enabled") + + # ------------------------------------------------------------------------- # Active modules list def has_module(self, module_name): diff --git a/private/templates/default/config.py b/private/templates/default/config.py index f8895b6420..f3291f223b 100644 --- a/private/templates/default/config.py +++ b/private/templates/default/config.py @@ -335,6 +335,10 @@ # Terms of Service to be able to Register on the system #settings.options.terms_of_service = T("Terms of Service\n\nYou have to be eighteen or over to register as a volunteer.") +#Message Parser Settings +#Add the enabled parsing workflows to the list below +#settings.parser.parser_enabled = ["parser_1", "parser_2", "parser_3"] + # Comment/uncomment modules here to disable/enable them # @ToDo: Have the system automatically enable migrate if a module is enabled # Modules menu is defined in modules/eden/menu.py diff --git a/private/templates/default/parser.py b/private/templates/default/parser.py new file mode 100644 index 0000000000..4ed9d96b32 --- /dev/null +++ b/private/templates/default/parser.py @@ -0,0 +1,283 @@ +''' +This file contains the parsing workflows that define different parsing rules. +Copy this file to modules/parser.py +To add more workflows, define functions with a name "parse_%s" (%workflow_task_id) +==> workflow_task_id is defined in s3db.msg_workflow +(e.g. For workflow_task_id =1 ; we have parse_1() , for workflow_task_id =2 , we have +parse_2() and so on. + +@author: Ashwyn Sharma +''' +import datetime +import difflib +import string +import urllib +from urllib2 import urlopen + +from gluon import current +from gluon.html import * +from gluon.http import redirect + +from s3.s3utils import s3_debug,soundex + + + + +class S3Parsing(object): + """ + Message Parsing Framework. + """ + + # ------------------------------------------------------------------------- + @staticmethod + def parse_1(message=""): + """ + Parsing Workflow 1. + """ + + + if not message: + return None + + T = current.T + db = current.db + s3db = current.s3db + s3mgr = current.manager + + primary_keywords = ["get", "give", "show"] # Equivalent keywords in one list + contact_keywords = ["email", "mobile", "facility", "clinical", + "security", "phone", "status", "hospital", + "person", "organisation"] + + pkeywords = primary_keywords+contact_keywords + keywords = string.split(message) + pquery = [] + name = "" + reply = "" + for word in keywords: + match = None + for key in pkeywords: + if soundex(key) == soundex(word): + match = key + break + if match: + pquery.append(match) + else: + name = word + + # --------------------------------------------------------------------- + # Person Search [get name person phone email] + if "person" in pquery: + + table = s3db.pr_person + rows = db(table.id > 0).select(table.pe_id, + table.first_name, + table.middle_name, + table.last_name) + for row in rows: + result = [] + if (soundex(str(name)) == soundex(str(row.first_name))) or \ + (soundex(str(name)) == soundex(str(row.middle_name))) or \ + (soundex(str(name)) == soundex(str(row.last_name))): + presult = dict(name = row.first_name, id = row.pe_id) + result.append(presult) + break + + if len(result) > 1: + return T("Multiple Matches") + if len(result) == 1: + reply = result[0]["name"] + table = s3db.pr_contact + if "email" in pquery: + query = (table.pe_id == result[0]["id"]) & \ + (table.contact_method == "EMAIL") + recipient = db(query).select(table.value, + orderby = table.priority, + limitby=(0, 1)).first() + reply = "%s Email->%s" % (reply, recipient.value) + if "phone" in pquery: + query = (table.pe_id == result[0]["id"]) & \ + (table.contact_method == "SMS") + recipient = db(query).select(table.value, + orderby = table.priority, + limitby=(0, 1)).first() + reply = "%s Mobile->%s" % (reply, + recipient.value) + + if len(result) == 0: + return T("No Match") + + return reply + return "Please provide one of the keywords - person, hospital, organisation" + + # --------------------------------------------------------------------- + # --------------------------------------------------------------------- + + @staticmethod + def parse_2(message=""): + """ + Parsing Workflow 2. + """ + + if not message: + return None + + T = current.T + db = current.db + s3db = current.s3db + s3mgr = current.manager + + primary_keywords = ["get", "give", "show"] # Equivalent keywords in one list + contact_keywords = ["email", "mobile", "facility", "clinical", + "security", "phone", "status", "hospital", + "person", "organisation"] + + pkeywords = primary_keywords+contact_keywords + keywords = string.split(message) + pquery = [] + name = "" + reply = "" + for word in keywords: + match = None + for key in pkeywords: + if soundex(key) == soundex(word): + match = key + break + if match: + pquery.append(match) + else: + name = word + + # --------------------------------------------------------------------- + # Hospital Search [example: get name hospital facility status ] + if "hospital" in pquery: + table = s3db.hms_hospital + rows = db(table.id > 0).select(table.id, + table.name, + table.aka1, + table.aka2) + for row in rows: + result = [] + if (soundex(str(name)) == soundex(str(row.name))) or \ + (soundex(name) == soundex(str(row.aka1))) or \ + (soundex(name) == soundex(str(row.aka2))): + result.append(row) + break + + + if len(result) > 1: + return T("Multiple Matches") + + if len(result) == 1: + hospital = db(table.id == result[0].id).select().first() + reply = "%s %s (%s) " % (reply, hospital.name, + T("Hospital")) + if "phone" in pquery: + reply = reply + "Phone->" + str(hospital.phone_emergency) + if "facility" in pquery: + reply = reply + "Facility status " + str(table.facility_status.represent(hospital.facility_status)) + if "clinical" in pquery: + reply = reply + "Clinical status " + str(table.clinical_status.represent(hospital.clinical_status)) + if "security" in pquery: + reply = reply + "Security status " + str(table.security_status.represent(hospital.security_status)) + + if len(result) == 0: + return T("No Match") + + return reply + return "Please provide one of the keywords - person, hospital, organisation" + + # --------------------------------------------------------------------- + # --------------------------------------------------------------------- + @staticmethod + def parse_3(message=""): + """ + Parsing Workflow 3. + """ + + if not message: + return None + + T = current.T + db = current.db + s3db = current.s3db + s3mgr = current.manager + + primary_keywords = ["get", "give", "show"] # Equivalent keywords in one list + contact_keywords = ["email", "mobile", "facility", "clinical", + "security", "phone", "status", "hospital", + "person", "organisation"] + + pkeywords = primary_keywords+contact_keywords + keywords = string.split(message) + pquery = [] + name = "" + reply = "" + for word in keywords: + match = None + for key in pkeywords: + if soundex(key) == soundex(word): + match = key + break + if match: + pquery.append(match) + else: + name = word + + # --------------------------------------------------------------------- + # Organization search [example: get name organisation phone] + if "organisation" in pquery: + table = s3db.org_organisation + rows = db(table.id > 0).select(table.id, + table.name, + table.acronym) + for row in rows: + result = [] + if (soundex(str(name)) == soundex(str(row.name))) or \ + (soundex(str(name)) == soundex(str(row.acronym))): + result.append(row) + break + + if len(result) > 1: + return T("Multiple Matches") + + if len(result) == 1: + organisation = db(table.id == result[0].id).select().first() + reply = "%s %s (%s) " % (reply, organisation.name, + T("Organization")) + if "phone" in pquery: + reply = reply + "Phone->" + str(organisation.donation_phone) + if "office" in pquery: + reply = reply + "Address->" + s3_get_db_field_value(tablename = "org_office", + fieldname = "address", + look_up_value = organisation.id) + if len(reply) == 0: + return T("No Match") + + return reply + return "Please provide one of the keywords - person, hospital, organisation" + + + # --------------------------------------------------------------------- + # --------------------------------------------------------------------- + @staticmethod + def parser(workflow, message = ""): + """ + Parsing Workflow Filter. + Called by parse_import() in s3msg.py. + """ + + if workflow == 1: + return S3Parsing.parse_1(message) + + elif workflow == 2: + return S3Parsing.parse_2(message) + + elif workflow == 3: + return S3Parsing.parse_3(message) + else: + return None + +# END ========================================================================= + +