diff --git a/bin/cylc-broadcast b/bin/cylc-broadcast index 76c0546ea26..780f8f6a92d 100755 --- a/bin/cylc-broadcast +++ b/bin/cylc-broadcast @@ -202,7 +202,7 @@ def main(): name, point_string = TaskID.split(options.showtask) except ValueError: parser.error("TASKID must be " + TaskID.SYNTAX) - settings = pclient.broadcast('get', options.showtask) + settings = pclient.broadcast('get', task_id=options.showtask) padding = get_padding(settings) * ' ' if options.raw: print str(settings) @@ -212,7 +212,9 @@ def main(): if options.clear: modified_settings, bad_options = pclient.broadcast( - 'clear', options.point_strings, options.namespaces) + 'clear', point_strings=options.point_strings, + namespaces=options.namespaces + ) if modified_settings: print get_broadcast_change_report( modified_settings, is_cancel=True) @@ -220,7 +222,7 @@ def main(): if options.expire: modified_settings, bad_options = pclient.broadcast( - 'expire', options.expire) + 'expire', cutoff=options.expire) if modified_settings: print get_broadcast_change_report( modified_settings, is_cancel=True) @@ -249,7 +251,9 @@ def main(): validate(setting, SPEC['runtime']['__MANY__']) settings.append(setting) modified_settings, bad_options = pclient.broadcast( - 'clear', point_strings, namespaces, settings) + 'clear', point_strings=point_strings, + namespaces=namespaces, cancel_settings=settings + ) if modified_settings: print get_broadcast_change_report( modified_settings, is_cancel=True) @@ -270,7 +274,9 @@ def main(): validate(setting, SPEC['runtime']['__MANY__']) settings.append(setting) modified_settings, bad_options = pclient.broadcast( - 'put', point_strings, namespaces, settings) + 'put', point_strings=point_strings, + namespaces=namespaces, settings=settings + ) print get_broadcast_change_report(modified_settings) sys.exit(get_broadcast_bad_options_report(bad_options, is_set=True)) diff --git a/bin/cylc-hold b/bin/cylc-hold index 763c34f56c6..9b94432d6aa 100755 --- a/bin/cylc-hold +++ b/bin/cylc-hold @@ -66,11 +66,13 @@ def main(): options.port, options.db, my_uuid=options.set_uuid, print_uuid=options.print_uuid) if args: - items, compat = parser.parse_multitask_compat(options, args) - pclient.put_command('hold_task', items, compat) + items = parser.parse_multitask_compat(options, args) + pclient.put_command('hold_tasks', items=items) elif options.hold_point_string: pclient.put_command( - 'hold_after_point_string', options.hold_point_string) + 'hold_after_point_string', + point_string=options.hold_point_string + ) else: pclient.put_command('hold_suite') diff --git a/bin/cylc-insert b/bin/cylc-insert index 65840736777..602f603f462 100755 --- a/bin/cylc-insert +++ b/bin/cylc-insert @@ -68,14 +68,13 @@ def main(): if (options.multitask_compat and len(args) in [2, 3] and all(["/" not in arg for arg in args]) and all(["." not in arg for arg in args[1:]])): - items, compat = (args[0], args[1]) + items = [(args[0] + "." + args[1])] if len(args) == 3: options.stop_point_string = args[2] prompt( - 'Insert %s at %s in %s' % (items, compat, suite), - options.force) + 'Insert %s in %s' % (items, suite), options.force) else: - items, compat = (args, None) + items = args for i, item in enumerate(items): if not TaskID.is_valid_id_for_insert(item): sys.exit('ERROR: "%s": invalid task ID (argument %d)' % ( @@ -88,7 +87,9 @@ def main(): print_uuid=options.print_uuid) pclient.put_command( - 'insert_task', items, compat, None, options.stop_point_string) + 'insert_tasks', items=items, + stop_point_string=options.stop_point_string + ) if __name__ == "__main__": diff --git a/bin/cylc-kill b/bin/cylc-kill index 556bf0015bd..db433d66004 100755 --- a/bin/cylc-kill +++ b/bin/cylc-kill @@ -55,8 +55,8 @@ def main(): suite, options.owner, options.host, options.pyro_timeout, options.port, options.db, my_uuid=options.set_uuid, print_uuid=options.print_uuid) - items, compat = parser.parse_multitask_compat(options, args) - pclient.put_command('kill_tasks', items, compat) + items = parser.parse_multitask_compat(options, args) + pclient.put_command('kill_tasks', items=items) if __name__ == "__main__": diff --git a/bin/cylc-ping b/bin/cylc-ping index 4df62f2455e..d25fbb1d514 100755 --- a/bin/cylc-ping +++ b/bin/cylc-ping @@ -31,7 +31,7 @@ if '--use-ssh' in sys.argv[1:]: import cylc.flags from cylc.CylcOptionParsers import cop from cylc.task_id import TaskID -from cylc.network.suite_info import SuiteInfoClient +from cylc.network.suite_info import SuiteInfoClientAnon from cylc.cfgspec.globalcfg import GLOBAL_CFG @@ -55,11 +55,10 @@ def main(): suite = args[0] - pclient = SuiteInfoClient( + pclient = SuiteInfoClientAnon( suite, options.owner, options.host, options.pyro_timeout, options.port, options.db, my_uuid=options.set_uuid, print_uuid=options.print_uuid) - pclient.set_use_scan_hash() # cylc ping SUITE pclient.get_info('ping_suite') # (no need to check the result) @@ -72,7 +71,7 @@ def main(): task_id = args[1] if not TaskID.is_valid_id(task_id): sys.exit("Invalid task ID: " + task_id) - success, msg = pclient.get_info('ping_task', task_id) + success, msg = pclient.get_info('ping_task', task_id=task_id) if not success: sys.exit('ERROR: ' + msg) diff --git a/bin/cylc-poll b/bin/cylc-poll index d7a5afb7662..6c4813a8a1e 100755 --- a/bin/cylc-poll +++ b/bin/cylc-poll @@ -62,8 +62,8 @@ def main(): suite, options.owner, options.host, options.pyro_timeout, options.port, options.db, my_uuid=options.set_uuid, print_uuid=options.print_uuid) - items, compat = parser.parse_multitask_compat(options, args) - pclient.put_command('poll_tasks', items, compat) + items = parser.parse_multitask_compat(options, args) + pclient.put_command('poll_tasks', items=items) if __name__ == "__main__": diff --git a/bin/cylc-release b/bin/cylc-release index c7c13286d42..ec1a444dca7 100755 --- a/bin/cylc-release +++ b/bin/cylc-release @@ -57,8 +57,8 @@ def main(): options.port, options.db, my_uuid=options.set_uuid, print_uuid=options.print_uuid) if args: - items, compat = parser.parse_multitask_compat(options, args) - pclient.put_command('release_task', items, compat) + items = parser.parse_multitask_compat(options, args) + pclient.put_command('release_tasks', items=items) else: pclient.put_command('release_suite') diff --git a/bin/cylc-remove b/bin/cylc-remove index 45943611946..05ce01700e4 100755 --- a/bin/cylc-remove +++ b/bin/cylc-remove @@ -57,8 +57,8 @@ def main(): suite, options.owner, options.host, options.pyro_timeout, options.port, options.db, my_uuid=options.set_uuid, print_uuid=options.print_uuid) - items, compat = parser.parse_multitask_compat(options, args) - pclient.put_command('remove_task', items, compat, options.spawn) + items = parser.parse_multitask_compat(options, args) + pclient.put_command('remove_tasks', items=items, spawn=options.spawn) if __name__ == "__main__": diff --git a/bin/cylc-reset b/bin/cylc-reset index dde00a52c10..44c0bbbc778 100755 --- a/bin/cylc-reset +++ b/bin/cylc-reset @@ -80,8 +80,9 @@ def main(): suite, options.owner, options.host, options.pyro_timeout, options.port, options.db, my_uuid=options.set_uuid, print_uuid=options.print_uuid) - items, compat = parser.parse_multitask_compat(options, args) - pclient.put_command('reset_task_state', items, compat, options.state) + items = parser.parse_multitask_compat(options, args) + pclient.put_command( + 'reset_task_states', items=items, state=options.state) if __name__ == "__main__": diff --git a/bin/cylc-set-runahead b/bin/cylc-set-runahead index 29eff6ba18d..71383d74b1e 100755 --- a/bin/cylc-set-runahead +++ b/bin/cylc-set-runahead @@ -59,7 +59,7 @@ def main(): if runahead: prompt('Change runahead limit in %s to %s' % (suite, runahead), options.force) - pclient.put_command('set_runahead', runahead) + pclient.put_command('set_runahead', interval=runahead) else: # no limit! prompt('Change runahead limit in %s to NO LIMIT' % suite, diff --git a/bin/cylc-set-verbosity b/bin/cylc-set-verbosity index b7a22ddb4b3..7091eef5691 100755 --- a/bin/cylc-set-verbosity +++ b/bin/cylc-set-verbosity @@ -70,7 +70,7 @@ def main(): options.port, options.db, my_uuid=options.set_uuid, print_uuid=options.print_uuid) - pclient.put_command('set_verbosity', priority) + pclient.put_command('set_verbosity', level=priority) if __name__ == "__main__": diff --git a/bin/cylc-show b/bin/cylc-show index cce7f4ded91..7a94c4738e4 100755 --- a/bin/cylc-show +++ b/bin/cylc-show @@ -30,7 +30,7 @@ if '--use-ssh' in sys.argv[1:]: sys.exit(0) import cylc.flags -from cylc.network.suite_info import SuiteInfoClient +from cylc.network.suite_info import SuiteInfoClient, SuiteInfoClientAnon from cylc.CylcOptionParsers import cop from cylc.task_id import TaskID @@ -47,7 +47,8 @@ def main(): pclient = SuiteInfoClient( suite, options.owner, options.host, options.pyro_timeout, options.port, options.db, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) + print_uuid=options.print_uuid + ) if len(args) == 1: # Print suite info. @@ -67,14 +68,16 @@ def main(): # Print task instance info. task_id = arg - info = pclient.get_info('get_task_info', name) + info = pclient.get_info('get_task_info', name=name) if not info: sys.exit("ERROR: task not found: %s" % name) for key, value in sorted(info.items(), reverse=True): print "%s: %s" % (key, value or "(not given)") if point_string is not None: - result = pclient.get_info('get_task_requisites', name, point_string) + result = pclient.get_info('get_task_requisites', + name=name, + point_string=point_string) if not result: sys.exit("ERROR: task instance not found: %s" % task_id) diff --git a/bin/cylc-spawn b/bin/cylc-spawn index a51c5c871c4..be18c6c4fa5 100755 --- a/bin/cylc-spawn +++ b/bin/cylc-spawn @@ -52,8 +52,8 @@ def main(): suite, options.owner, options.host, options.pyro_timeout, options.port, options.db, my_uuid=options.set_uuid, print_uuid=options.print_uuid) - items, compat = parser.parse_multitask_compat(options, args) - pclient.put_command('spawn_tasks', items, compat) + items = parser.parse_multitask_compat(options, args) + pclient.put_command('spawn_tasks', items=items) if __name__ == "__main__": diff --git a/bin/cylc-stop b/bin/cylc-stop index ad27abbedc2..02fb24d0b3e 100755 --- a/bin/cylc-stop +++ b/bin/cylc-stop @@ -59,8 +59,9 @@ class stop_poller(poller): def load(self): self.pclient = SuiteInfoClient( - self.args['suite'], self.args['owner'], self.args['host'], - self.args['pyro_timeout'], self.args['port'], self.args['db']) + self.args['suite'], owner=self.args['owner'], + host=self.args['host'], timeout=self.args['pyro_timeout'], + port=self.args['port'], db=self.args['db']) def check(self): # return True if suite has stopped (success) else False @@ -148,13 +149,19 @@ def main(): if method: prompt(prompt_text + ' for ' + suite, options.force) - pclient.put_command(method, shutdown_arg) + if method == 'set_stop_after_clock_time': + pclient.put_command(method, datetime_string=shutdown_arg) + if method == 'set_stop_after_task': + pclient.put_command(method, task_id=shutdown_arg) + if method == 'set_stop_after_point': + pclient.put_command(method, point_string=shutdown_arg) elif options.now: prompt('Shut down %s now' % suite, options.force) pclient.put_command('stop_now') else: prompt('Shut down %s' % suite, options.force) - pclient.put_command('set_stop_cleanly', options.kill) + pclient.put_command('set_stop_cleanly', + kill_active_tasks=options.kill) if int(options.max_polls) > 0: # (test to avoid the "nothing to do" warning for # --max-polls=0) diff --git a/bin/cylc-trigger b/bin/cylc-trigger index 9e689e7ff77..751965061dc 100755 --- a/bin/cylc-trigger +++ b/bin/cylc-trigger @@ -89,23 +89,21 @@ def main(): print_uuid=options.print_uuid) if options.edit_run: - items, compat = parser.parse_multitask_compat(options, args) - if compat: - task_id = TaskID.get(items, compat) - else: - task_id = args[0] + items = parser.parse_multitask_compat(options, args) + task_id = items[0] # Check that TASK is a unique task. info_client = SuiteInfoClient( suite, options.owner, options.host, options.pyro_timeout, options.port, options.db, my_uuid=cmd_client.my_uuid) - success, msg = info_client.get_info('ping_task', task_id, True) + success, msg = info_client.get_info( + 'ping_task', task_id=task_id, exists_only=True) if not success: sys.exit('ERROR: %s' % msg) # Get the job filename from the suite daemon - the task cycle point may # need standardising to the suite cycle point format. jobfile_path, compat = info_client.get_info( - 'get_task_jobfile_path', task_id) + 'get_task_jobfile_path', task_id=task_id) if not jobfile_path: sys.exit('ERROR: task not found') @@ -124,7 +122,7 @@ def main(): old_mtime = None # Tell the suite daemon to generate the job file. - cmd_client.put_command('dry_run_task', [task_id]) + cmd_client.put_command('dry_run_tasks', items=[task_id]) # Wait for the new job file to be written. Use mtime because the same # file could potentially exist already, left from a previous run. @@ -194,8 +192,8 @@ def main(): sys.exit(0) # Trigger the task proxy(s). - items, compat = parser.parse_multitask_compat(options, args) - cmd_client.put_command('trigger_task', items, compat) + items = parser.parse_multitask_compat(options, args) + cmd_client.put_command('trigger_tasks', items=items) if __name__ == "__main__": diff --git a/lib/Pyro/EventService/Clients.py b/lib/Pyro/EventService/Clients.py deleted file mode 100644 index 72bc734cb84..00000000000 --- a/lib/Pyro/EventService/Clients.py +++ /dev/null @@ -1,91 +0,0 @@ -############################################################################# -# -# Event Service client base classes -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -import Pyro.core, Pyro.naming, Pyro.constants -import Pyro.EventService.Server -from Pyro.EventService.Event import Event -from Pyro.errors import * - -# SUBSCRIBER: subscribes to certain events. -class Subscriber(Pyro.core.CallbackObjBase): - def __init__(self, ident=None, esURI=None): - Pyro.core.CallbackObjBase.__init__(self) - Pyro.core.initServer() - Pyro.core.initClient() - daemon = Pyro.core.Daemon() - if esURI: - check=Pyro.core.PyroURI(esURI) - self.ES_uri=esURI - else: - locator = Pyro.naming.NameServerLocator(identification=ident) - self.NS = locator.getNS(host=Pyro.config.PYRO_NS_HOSTNAME) - daemon.useNameServer(self.NS) - self.ES_uri = self.NS.resolve(Pyro.constants.EVENTSERVER_NAME) - daemon.connect(self) # will also set self.daemon... - self.ES_ident=ident - self.abortListen=0 - self.daemon=daemon # make sure daemon doesn't get garbage collected now - - def getES(self): - # we get a fresh proxy to the ES because of threading issues. - # (proxies can not be reused across multiple threads) - eventservice=Pyro.core.getProxyForURI(self.ES_uri) - eventservice._setIdentification(self.ES_ident) - return eventservice - - def subscribe(self,subjects): - # Subscribe to one or more subjects. - # It is safe to call this multiple times. - self.getES().subscribe(subjects, self.getProxy()) - def subscribeMatch(self,subjectPatterns): - # Subscribe to one or more subjects (by pattern) - # It is safe to call this multiple times. - self.getES().subscribeMatch(subjectPatterns, self.getProxy()) - def unsubscribe(self, subjects): - # Unsubscribe the subscriber for the given subject(s). - self.getES().unsubscribe(subjects, self.getProxy()) - - def abort(self): - self.abortListen=1 - - def setThreading(self, threaded): - self.getDaemon().threaded=threaded - - def listen(self): - self.getDaemon().requestLoop(lambda s=self: not s.abortListen) - - def event(self, event): # callback, override this! - print event - -# PUBLISHER: publishes events. -class Publisher(object): - def __init__(self, ident=None, esURI=None): - Pyro.core.initClient() - if esURI: - check=Pyro.core.PyroURI(esURI) - self.ES_uri=esURI - else: - locator = Pyro.naming.NameServerLocator(identification=ident) - ns = locator.getNS(host=Pyro.config.PYRO_NS_HOSTNAME) - self.ES_uri = ns.resolve(Pyro.constants.EVENTSERVER_NAME) - ns._release() # be very sure to release the socket - self.ES_ident = ident - - def getES(self): - # we get a fresh proxy to the ES because of threading issues. - # (proxies can not be reused across multiple threads) - eventservice=Pyro.core.getProxyForURI(self.ES_uri) - eventservice._setIdentification(self.ES_ident) - return eventservice - - def publish(self, subjects, msg): - es=self.getES() - es.publish(subjects,msg) - es._release() # be very sure to release the socket - diff --git a/lib/Pyro/EventService/Event.py b/lib/Pyro/EventService/Event.py deleted file mode 100644 index d5e411d7c27..00000000000 --- a/lib/Pyro/EventService/Event.py +++ /dev/null @@ -1,19 +0,0 @@ -############################################################################# -# -# Event Service client base classes -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -import time - -# EVENT - the thing that is published. Has a subject and contains a message. -class Event(object): - def __init__(self, subject, msg, creationTime=None): - self.msg=msg - self.subject=subject - self.time=creationTime or time.time() - def __str__(self): - return "" % (self.subject, time.ctime(self.time), str(self.msg)) diff --git a/lib/Pyro/EventService/Server.py b/lib/Pyro/EventService/Server.py deleted file mode 100644 index 8b23dfc252a..00000000000 --- a/lib/Pyro/EventService/Server.py +++ /dev/null @@ -1,301 +0,0 @@ -############################################################################# -# -# Event Service daemon and server classes -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -import time, types, re, sys, traceback, os -import Pyro.core, Pyro.naming, Pyro.util, Pyro.constants -from Pyro.errors import * -from Pyro.EventService.Event import Event -import Queue -from threading import Thread - -Log=Pyro.util.Log - -# SUBSCRIBER - each subscriber has one of these worker threads -class Subscriber(Thread): - def __init__(self, remote): - Thread.__init__(self) - self.remote=remote - # set the callback method to ONEWAY mode: - self.remote._setOneway("event") - self.queue=Queue.Queue(Pyro.config.PYRO_ES_QUEUESIZE) - def run(self): - while 1: - event=self.queue.get() - if isinstance(event,Event): - try: - self.remote.event(event) - except ProtocolError,x: - break - else: - break # it was no Event, so exit - # this reads all pending items from the queue so that any - # tasks that are blocked on the queue can continue. - (queue, self.queue) = (self.queue, None) - try: - while 1: - queue.get(block=0) - except Queue.Empty: - pass - # release the remote connection - self.remote._release() - del self.remote - def send(self, event): - if self.queue: - self.queue.put(event, block=Pyro.config.PYRO_ES_BLOCKQUEUE) - def running(self): - return self.queue - -# The EVENTSERVICE is the actual Pyro server. -# -# BTW: Subscribers are remembered trough their proxy class. -# This class is capable of being a correct key in a dictionary. -class EventService(Pyro.core.ObjBase): - def __init__(self): - Pyro.core.ObjBase.__init__(self) - self.subscribers={} # subject -> { threadname-> subscriberthread } - self.subscribersMatch={} # subjectPattern -> { threadname->subscriberthread } - self.subscriptionWorkers={} # subscriber -> subscription thread object - def _mksequence(self, seq): - if not (type(seq) in (types.TupleType,types.ListType)): - return (seq,) - return seq - def getSubscriptionWorker(self, subscriber): - # If this subscriber doesn't have its own subscription thread, create one. - if subscriber not in self.subscriptionWorkers: - worker = Subscriber(subscriber) - worker.start() - self.subscriptionWorkers[subscriber]=worker - return worker - else: - return self.subscriptionWorkers[subscriber] - def subscribe(self, subjects, subscriber): - if not subjects: return - # Subscribe into a dictionary; this way; somebody can subscribe - # only once to this subject. Subjects are exact strings. - for subject in self._mksequence(subjects): - worker=self.getSubscriptionWorker(subscriber) - self.subscribers.setdefault(subject.lower(),{}) [worker.getName()]=worker - def subscribeMatch(self, subjects, subscriber): - if not subjects: return - # Subscribe into a dictionary; this way; somebody can subscribe - # only once to this subject. Subjects are regex patterns. - for subject in self._mksequence(subjects): - worker=self.getSubscriptionWorker(subscriber) - matcher = re.compile(subject,re.IGNORECASE) - self.subscribersMatch.setdefault(matcher,{}) [worker.getName()]=worker - def unsubscribe(self, subjects, subscriber): - if not subjects: return - for subject in self._mksequence(subjects): - try: - blaat=self.subscribers[subject.lower()] # check for subject - worker=self.subscriptionWorkers[subscriber] - del self.subscribers[subject.lower()] [worker.getName()] - self.killWorkerIfLastSubject(subscriber, worker) - except KeyError,x: - try: - m=re.compile(subject,re.IGNORECASE) - worker=self.subscriptionWorkers[subscriber] - del self.subscribersMatch[m] [worker.getName()] - self.killWorkerIfLastSubject(subscriber,worker) - except KeyError,x: - pass - - def publish(self, subjects, message): - if not subjects: return - # keep the creation time, this must be the same for all events. - creationTime=time.time() - # publish a message. Subjects must be exact strings - for subject in self._mksequence(subjects): - event = Event(subject, message, creationTime) - subjectLC=subject.lower() - try: - for (name,s) in self.subscribers[subjectLC].items(): - try: - if s.running(): - s.send(event) - else: - try: - del self.subscribers[subjectLC][name] - except KeyError: - pass - except Queue.Full: - pass - except KeyError: - pass - # process the subject patterns - for (m,subs) in self.subscribersMatch.items(): - if m.match(subject): - # send event to all subscribers - for (name,s) in subs.items(): - try: - if s.running(): - s.send(event) - else: - try: - del subs[name] - except KeyError: - pass - except Queue.Full: - pass - - def killWorkerIfLastSubject(self, subscriber, worker): - item=(worker.getName(),worker) - for v in self.subscribers.values(): - if item in v.items(): - return - for v in self.subscribersMatch.values(): - if item in v.items(): - return - worker.send("QUIT") - del self.subscriptionWorkers[subscriber] - - -class EventServiceStarter(object): - def __init__(self, identification=None): - Pyro.core.initServer() - self.running=1 - self.identification=identification - self.started = Pyro.util.getEventObject() - def start(self, *args, **kwargs): # see _start for allowed arguments - kwargs["startloop"]=1 - self._start(*args, **kwargs ) - def initialize(self, *args, **kwargs): # see _start for allowed arguments - kwargs["startloop"]=0 - self._start( *args, **kwargs ) - def getServerSockets(self): - return self.daemon.getServerSockets() - def waitUntilStarted(self,timeout=None): - self.started.wait(timeout) - return self.started.isSet() - def _start(self,hostname='',port=None,startloop=1,useNameServer=1,norange=0): - daemon = Pyro.core.Daemon(host=hostname,port=port,norange=norange) - if self.identification: - daemon.setAllowedIdentifications([self.identification]) - print 'Requiring connection authentication.' - - if useNameServer: - locator = Pyro.naming.NameServerLocator(identification=self.identification) - ns = locator.getNS() - - # check if ES already running - try: - ns.resolve(Pyro.constants.EVENTSERVER_NAME) - print 'The Event Server appears to be already running.' - print 'You cannot start multiple Event Servers.' - ans=raw_input('Start new Event Server anyway (y/n)? ') - if ans!='y': - return - ns.unregister(Pyro.constants.EVENTSERVER_NAME) - except NamingError: - pass - - daemon.useNameServer(ns) - - es = EventService() - - esURI=daemon.connect(es, Pyro.constants.EVENTSERVER_NAME) - print 'URI=',esURI - - message = daemon.validateHostnameAndIP() - if message: - print "\nWARNING:",message,"\n" - - print 'Event Server started.' - - self.started.set() # signal that we've started. - - if startloop: - Log.msg('ES daemon','This is the Pyro Event Server.') - - try: - if os.name!="java": - # I use a timeout here otherwise you can't break gracefully on Windows - daemon.setTimeout(20) - daemon.requestLoop(lambda s=self: s.running) - except KeyboardInterrupt: - Log.warn('ES daemon','shutdown on user break signal') - print 'Shutting down on user break signal.' - self.shutdown(es) - except: - try: - (exc_type, exc_value, exc_trb) = sys.exc_info() - out = ''.join(traceback.format_exception(exc_type, exc_value, exc_trb)[-5:]) - Log.error('ES daemon', 'Unexpected exception, type',exc_type, - '\n--- partial traceback of this exception follows:\n', - out,'\n--- end of traceback') - print '*** Exception occured!!! Partial traceback:' - print out - print '*** Resuming operations...' - finally: - del exc_type, exc_value, exc_trb # delete refs to allow proper GC - - Log.msg('ES daemon','Shut down gracefully.') - print 'Event Server gracefully stopped.' - else: - # no loop, store the required objects for getServerSockets() - self.daemon=daemon - self.es=es - if os.name!="java": - daemon.setTimeout(20) # XXX fixed timeout - - def mustContinueRunning(self): - return self.running - def handleRequests(self, timeout=None): - # this method must be called from a custom event loop - self.daemon.handleRequests(timeout=timeout) - def shutdown(self,es): - if es: - # internal shutdown call with specified ES object - daemon=es.getDaemon() - else: - # custom shutdown call w/o specified ES object, use stored instance - daemon=self.daemon - es=self.es - del self.es, self.daemon - try: - daemon.disconnect(es) # clean up nicely - except NamingError,x: - Log.warn('ES daemon','disconnect error during shutdown:',x) - except ConnectionClosedError,x: - Log.warn('ES daemon','lost connection with Name Server, cannot unregister') - self.running=0 - daemon.shutdown() - - -def start(argv): - Args = Pyro.util.ArgParser() - Args.parse(argv,'hNn:p:i:') - if Args.hasOpt('h'): - print 'Usage: pyro-es [-h] [-n hostname] [-p port] [-N] [-i identification]' - print ' where -p = ES server port (0 for auto)' - print ' -n = non-default hostname to bind on' - print ' -N = do not use the name server' - print ' -i = the required authentication ID for ES clients,' - print ' also used to connect to other Pyro services' - print ' -h = print this help' - raise SystemExit - hostname = Args.getOpt('n',None) - port = Args.getOpt('p',None) - useNameServer = not Args.hasOpt('N') - ident = Args.getOpt('i',None) - if port: - port=int(port) - norange=(port==0) - Args.printIgnored() - if Args.args: - print 'Ignored arguments:',' '.join(Args.args) - - print '*** Pyro Event Server ***' - starter=EventServiceStarter(identification=ident) - starter.start(hostname,port,useNameServer=useNameServer,norange=norange) - - -# allow easy starting of the ES by using python -m -if __name__=="__main__": - start(sys.argv[1:]) diff --git a/lib/Pyro/EventService/__init__.py b/lib/Pyro/EventService/__init__.py deleted file mode 100644 index 6f66f15aed2..00000000000 --- a/lib/Pyro/EventService/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# just to make this a package. diff --git a/lib/Pyro/__init__.py b/lib/Pyro/__init__.py deleted file mode 100644 index f241e188ed5..00000000000 --- a/lib/Pyro/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -############################################################################# -# -# Pyro file to make Pyro a package, and to set up configuration. -# -# This is part of "Pyro" - Python Remote Objects -# Which is (c) Irmen de Jong - irmen@razorvine.net -# -# Note: to see what Pyro version this is, print Pyro.constants.VERSION -# -############################################################################# - - -# Initialize Pyro Configuration. -# -# This is put here because it could actually initialize config stuff needed -# even before the code calls core.initClient or core.initServer. -# -# Pyro.config is a class, which has a __getattr__ member, so all -# pyro code can use Pyro.config. to look up a value. -# This allows for tweaking the configuration lookups by writing -# a custom __getattr__ and/or __init__ for the class. -# However, currently the class initializer adds configuration items -# as regular class data members. - -import os -import Pyro.configuration - -config = Pyro.configuration.Config() -try: - confFile = os.environ['PYRO_CONFIG_FILE'] -except KeyError: - confFile = '' -if not confFile and os.path.isfile('Pyro.conf'): - confFile='Pyro.conf' -config.setup(confFile) diff --git a/lib/Pyro/configuration.py b/lib/Pyro/configuration.py deleted file mode 100644 index 0489a701313..00000000000 --- a/lib/Pyro/configuration.py +++ /dev/null @@ -1,238 +0,0 @@ -############################################################################# -# -# Sets up Pyro's configuration (Pyro.config). -# -# This is part of "Pyro" - Python Remote Objects -# Which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - - -# Initialize Pyro Configuration. - -import re, os, random, tempfile -import Pyro.errors -from Pyro.errors import PyroError -import Pyro.constants -import Pyro.util2 # not util because of cyclic dependency - -try: - from pickle import HIGHEST_PROTOCOL as PICKLE_HIGHEST_PROTOCOL -except ImportError: - PICKLE_HIGHEST_PROTOCOL = 1 - - -# ---------------------- DEFAULT CONFIGURATION VARIABLES ----------- - -# Special characters are '$CURDIR' (current directory, absolute) and -# $STORAGE which is replaced by the PYRO_STORAGE path. -_defaults= { - 'PYRO_STORAGE': '$CURDIR', # current dir (abs) - 'PYRO_HOST': '', - 'PYRO_PUBLISHHOST': None, - 'PYRO_PORT': 7766, - 'PYRO_PORT_RANGE': 100, - 'PYRO_NS_HOSTNAME': None, - 'PYRO_NS_PORT': 9090, # tcp - 'PYRO_NS_BC_ADDR': None, - 'PYRO_NS_BC_PORT': 9090, # udp - 'PYRO_NS2_HOSTNAME': None, - 'PYRO_NS2_PORT': 9091, # tcp - 'PYRO_NS2_BC_ADDR': None, - 'PYRO_NS2_BC_PORT': 9091, # udp - 'PYRO_NS_URIFILE': '$STORAGE/Pyro_NS_URI', # (abs) - 'PYRO_NS_DEFAULTGROUP': ':Default', - 'PYRO_BC_RETRIES': 1, - 'PYRO_BC_TIMEOUT': 0.75, - 'PYRO_PICKLE_FORMAT': PICKLE_HIGHEST_PROTOCOL, - 'PYRO_XML_PICKLE': None, - 'PYRO_GNOSIS_PARANOIA': 0, - 'PYRO_TRACELEVEL': 0, - 'PYRO_USER_TRACELEVEL': 0, - 'PYRO_LOGFILE': '$STORAGE/Pyro_log', # (abs) - 'PYRO_USER_LOGFILE': '$STORAGE/Pyro_userlog', # (abs) - 'PYRO_STDLOGGING': 0, - 'PYRO_STDLOGGING_CFGFILE': 'logging.cfg', - 'PYRO_MAXCONNECTIONS': 200, - 'PYRO_TCP_LISTEN_BACKLOG': 200, - 'PYRO_BROKEN_MSGWAITALL': 0, - 'PYRO_MULTITHREADED': 1, # assume 1 - 'PYRO_COMPRESSION': 0, - 'PYRO_MOBILE_CODE': 0, - 'PYRO_DNS_URI': 0, - 'PYRO_CHECKSUM': 0, - 'PYRO_SOCK_KEEPALIVE': 1, - 'PYRO_ES_QUEUESIZE': 1000, - 'PYRO_ES_BLOCKQUEUE': 1, - 'PYRO_DETAILED_TRACEBACK': 0, - 'PYRO_ONEWAY_THREADED': 1, - 'PYROSSL_CERTDIR': '$STORAGE/certs', # (abs) - 'PYROSSL_CA_CERT': 'ca.pem', - 'PYROSSL_CERT': 'host.pem', - 'PYROSSL_KEY': None, - 'PYROSSL_POSTCONNCHECK': 1 -} - -# ---------------------- END OF DEFAULT CONFIGURATION VARIABLES ----- - - -class Config(object): - - def __init__(self): - _defaults['PYRO_MULTITHREADED']=Pyro.util2.supports_multithreading() - self.__dict__[Pyro.constants.CFGITEM_PYRO_INITIALIZED] = 0 - - def __eq__(self, other): - return self.__dict__==other.__dict__ - - def setup(self, configFile): - reader = ConfigReader(_defaults) - try: - reader.parse(configFile) - except EnvironmentError,x: - raise PyroError("Error reading config file: "+configFile+"; "+str(x)); - self.__dict__.update(reader.items) - if configFile: - self.__dict__['PYRO_CONFIG_FILE'] = os.path.abspath(configFile) - else: - self.__dict__['PYRO_CONFIG_FILE'] = '' - - def finalizeConfig_Client(self): - # For the client, we're done for now! - # It's nice if the storage directory exists and is - # writable, but if it isn't, we can continue happily. - # If Pyro needs to write something (log?), it will - # fail at that point if it can't access the storage dir. - # This behavior is good enough for clients. - pass - - def finalizeConfig_Server(self, storageCheck): - if storageCheck: - # The server needs a storage dir. Because it's a server, - # this usually is no problem. So create & test it here. - # Create the storage directory if it doesn't exist yet - if not os.path.exists(self.PYRO_STORAGE): - os.mkdir(self.PYRO_STORAGE) - # see if we have permission there, in a thread-safe fashion. - if not os.path.isdir(self.PYRO_STORAGE): - raise IOError('PYRO_STORAGE is not a directory ['+self.PYRO_STORAGE+']') - - try: - if os.name=='java': - # jython doesn't have suitable TemporaryFile implementation (lacks dir param) - javatestfile=os.path.join(self.PYRO_STORAGE,'_pyro_'+str(random.random())+".tmp") - f=open(javatestfile,"w") - else: - # use tempfile to safely create a unique temporary file even on multi-cpu nodes - f=tempfile.TemporaryFile(dir=self.PYRO_STORAGE, suffix='.tmp', prefix='_pyro_') - except Exception,x: - print x - raise IOError('no write access to PYRO_STORAGE ['+self.PYRO_STORAGE+']') - else: - f.close() - if os.name=='java': - os.remove(javatestfile) - -# def __getattr__(self,name): -# # add smart code here to deal with other requested config items! - - - -class ConfigReader(object): - def __init__(self, defaults): - self.matcher=re.compile(r'^(\w+)\s*=\s*(\S*)') - self.items=defaults.copy() - - def _check(self, filename): - print "ConfigReader: checking file", filename - items=[] - for l in open(filename).readlines(): - l=l.rstrip() - if not l or l.startswith('#'): - continue # skip empty line or comment - match=self.matcher.match(l) - if match: - items.append(match.group(1)) - allitems=self.items.keys() - allitems.sort() - for item in allitems: - if item not in items: - print "MISSING item: ",item - try: - items.remove(item) - except ValueError: - pass - if items: - print "items NOT in DEFAULTS:", items - else: - print "ok!" - - - def parse(self, filename): - linenum=0 - if filename: - for l in open(filename).readlines(): - l=l.rstrip() - linenum=linenum+1 - if not l or l.startswith('#'): - continue # skip empty line or comment - match=self.matcher.match(l) - if match: - if match.group(1) in _defaults.keys(): - if match.group(2): - self.items[match.group(1)] = match.group(2) - else: - raise KeyError('Unknown config item in configfile (line %d): %s' % (linenum, match.group(1))) - else: - raise ValueError('Syntax error in config file, line '+str(linenum)) - - # Parse the environment variables (they override the config file) - self.items.update(self.processEnv(_defaults.keys())) - - # First, fix up PYRO_STORAGE because others depend on it. - self.items['PYRO_STORAGE'] = self.treatSpecial(self.items['PYRO_STORAGE']) - # Now fix up all other items: - for i in self.items.keys(): - newVal = self.treatSpecial(self.items[i]) - if i in ('PYRO_STORAGE', 'PYRO_LOGFILE', 'PYRO_USER_LOGFILE', 'PYRO_NS_URIFILE'): - newVal=os.path.abspath(newVal) - # fix the variable type if it's an integer or float - if type(_defaults[i]) == type(42): - newVal = int(newVal) - if type(_defaults[i]) == type(0.1): - newVal = float(newVal) - self.items[i]= newVal - - def processEnv(self, keys): - env={} - for key in keys: - try: env[key] = os.environ[key] - except KeyError: pass - return env - - def treatSpecial(self, value): - # treat special escape strings - if type(value)==type(""): - if value=='$CURDIR': - return os.curdir - elif value.startswith('$STORAGE/'): - return os.path.join(self.items['PYRO_STORAGE'], value[9:]) - return value - -# easy config diagnostic with python -m -if __name__=="__main__": - print "Pyro version:",Pyro.constants.VERSION - r=ConfigReader(_defaults) - if os.path.exists("Pyro.conf"): - r._check("Pyro.conf") - x=Config() - if os.path.exists("Pyro.conf"): - x.setup("Pyro.conf") - else: - x.setup(None) - x.finalizeConfig_Server(1) - items=vars(x).items() - items.sort() - print "Active configuration settings:" - for item,value in items: - print item+"="+str(value) diff --git a/lib/Pyro/constants.py b/lib/Pyro/constants.py deleted file mode 100644 index 27765726580..00000000000 --- a/lib/Pyro/constants.py +++ /dev/null @@ -1,52 +0,0 @@ -############################################################################# -# -# Pyro global constants -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - - -# General Pyro Version String #### -VERSION = '3.16' - -# Fixed (internal) GUIDs -INTERNAL_DAEMON_GUID='c0000000'+'01100000'+'10000000'+'10000001' - -# Pyro names for the standard Services -NAMESERVER_NAME = ":Pyro.NameServer" -EVENTSERVER_NAME = ":Pyro.EventService" - -# Pyro traceback attribute for remote exceptions -TRACEBACK_ATTRIBUTE = "remote_stacktrace" - - -#### Remote Invocation Flags (bit flags) #### - -RIF_Varargs = (1<<0) # for '*args' syntax -RIF_Keywords = (1<<1) # for '**keywords' syntax -RIF_Oneway = (1<<2) # for oneway (no result) messages - currently internal use only -RIF_VarargsAndKeywords = RIF_Varargs | RIF_Keywords - - -#### Reasons why a connection may be denied #### -DENIED_UNSPECIFIED=0 -DENIED_SERVERTOOBUSY=1 -DENIED_HOSTBLOCKED=2 -DENIED_SECURITY=3 - -deniedReasons={ - DENIED_UNSPECIFIED:'unspecified reason', - DENIED_SERVERTOOBUSY:'server too busy', - DENIED_HOSTBLOCKED:'host blocked', - DENIED_SECURITY:'security reasons' - } - -# special config items -CFGITEM_PYRO_INITIALIZED = "_PYRO_INITIALIZED" - -# NS roles -NSROLE_SINGLE=0 -NSROLE_PRIMARY=1 -NSROLE_SECONDARY=2 diff --git a/lib/Pyro/core.py b/lib/Pyro/core.py deleted file mode 100644 index aa45f56cf60..00000000000 --- a/lib/Pyro/core.py +++ /dev/null @@ -1,868 +0,0 @@ -############################################################################# -# -# Pyro Core Library -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -from __future__ import with_statement -import sys, time, re, os, weakref -import imp, marshal, new, socket -from pickle import PicklingError -import Pyro.constants, Pyro.util, Pyro.protocol, Pyro.errors -from Pyro.errors import * -from types import UnboundMethodType, MethodType, BuiltinMethodType, TupleType, StringType, UnicodeType -if Pyro.util.supports_multithreading(): - import threading - -Log=Pyro.util.Log - - -def _checkInit(pyrotype="client"): - if not getattr(Pyro.config, Pyro.constants.CFGITEM_PYRO_INITIALIZED): - # If Pyro has not been initialized explicitly, do it automatically. - if pyrotype=="server": - initServer() - else: - initClient() - - -############################################################################# -# -# ObjBase - Server-side object implementation base class -# or master class with the actual object as delegate -# -# SynchronizedObjBase - Just the same, but with synchronized method -# calls (thread-safe). -# -############################################################################# - -class ObjBase(object): - def __init__(self): - self.objectGUID=Pyro.util.getGUID() - self.delegate=None - self.lastUsed=time.time() # for later reaping unused objects - if Pyro.config.PYRO_MOBILE_CODE: - self.codeValidator=lambda n,m,a: 1 # always accept - def GUID(self): - return self.objectGUID - def setGUID(self, guid): # used with persistent name server - self.objectGUID = guid - def delegateTo(self,delegate): - self.delegate=delegate - def setPyroDaemon(self, daemon): - # This will usually introduce a cyclic reference between the - # object and the daemon. Use a weak ref if available. - # NOTE: if you correctly clean up the object (that is, disconnect it from the daemon) - # the cyclic reference is cleared correctly, and no problem occurs. - # NOTE: you have to make sure your original daemon object doesn't get garbage collected - # if you still want to use the objects! You have to keep a ref. to the daemon somewhere. - if daemon: - self.daemon=weakref.proxy(daemon) - else: - self.daemon=None - def setCodeValidator(self, v): - if not callable(v): - raise TypeError("codevalidator must be a callable object") - self.codeValidator=v - def getDaemon(self): - return self.daemon - def getLocalStorage(self): - return self.daemon.getLocalStorage() - def _gotReaped(self): - # Called when daemon reaps this object due to unaccessed time - # Override this method if needed; to act on this event - pass - def getProxy(self): - return self.daemon.getProxyForObj(self) - def getAttrProxy(self): - return self.daemon.getAttrProxyForObj(self) - def Pyro_dyncall(self, method, flags, args): - # update the timestamp - self.lastUsed=time.time() - # find the method in this object, and call it with the supplied args. - keywords={} - if flags & Pyro.constants.RIF_Keywords: - # reconstruct the varargs from a tuple like - # (a,b,(va1,va2,va3...),{kw1:?,...}) - keywords=args[-1] - args=args[:-1] - if flags & Pyro.constants.RIF_Varargs: - # reconstruct the varargs from a tuple like (a,b,(va1,va2,va3...)) - args=args[:-1]+args[-1] - if keywords and type(keywords.iterkeys().next()) is unicode and sys.platform!="cli": - # IronPython sends all strings as unicode, but apply() doesn't grok unicode keywords. - # So we need to rebuild the keywords dict with str keys... - keywords = dict([(str(k),v) for k,v in keywords.iteritems()]) - # If the method is part of ObjBase, never call the delegate object because - # that object doesn't implement that method. If you don't check this, - # remote attributes won't work with delegates for instance, because the - # delegate object doesn't implement _r_xa. (remote_xxxattr) - if method in dir(ObjBase): - return getattr(self,method) (*args,**keywords) - else: - # try..except to deal with obsoleted string exceptions (raise "blahblah") - try : - return getattr(self.delegate or self,method) (*args,**keywords) - except : - exc_info = sys.exc_info() - try: - if type(exc_info[0]) == StringType : - if exc_info[1] == None : - raise Exception, exc_info[0], exc_info[2] - else : - raise Exception, "%s: %s" % (exc_info[0], exc_info[1]), exc_info[2] - else : - raise - finally: - del exc_info # delete frame to allow proper GC - - # remote getattr/setattr support: - def _r_ha(self, attr): - try: - attr = getattr(self.delegate or self,attr) - if type(attr) in (UnboundMethodType, MethodType, BuiltinMethodType): - return 1 # method - except: - pass - return 2 # attribute - def _r_ga(self, attr): - return getattr(self.delegate or self, attr) - def _r_sa(self, attr, value): - setattr(self.delegate or self, attr, value) - # remote code downloading support (server downloads from client): - def remote_supply_code(self, name, module, sourceaddr): - # XXX this is nasty code, and also duplicated in protocol.py _retrieveCode() - if Pyro.config.PYRO_MOBILE_CODE and self.codeValidator(name,module,sourceaddr): - try: - imp.acquire_lock() # threadsafe imports - if name in sys.modules and getattr(sys.modules[name],'_PYRO_bytecode',None): - # already have this module, don't import again - # we checked for the _PYRO_bytecode attribute because that is only - # present when all loading code below completed successfully - return - Log.msg('ObjBase','loading supplied code: ',name,'from',str(sourceaddr)) - if module[0:4]!=imp.get_magic(): - # compile source code - code=compile(module,'','exec') - else: - # read bytecode from the client - code=marshal.loads(module[8:]) - - # make the module hierarchy and add all names to sys.modules - name=name.split('.') - path='' - mod=new.module("pyro-agent-context") - for m in name: - path+='.'+m - # use already loaded modules instead of overwriting them - real_path = path[1:] - if sys.modules.has_key(real_path): - mod = sys.modules[real_path] - else: - setattr(mod,m,new.module(path[1:])) - mod=getattr(mod,m) - sys.modules[path[1:]]=mod - # execute the module code in the right module. - exec code in mod.__dict__ - # store the bytecode for possible later reference if we need to pass it on - mod.__dict__['_PYRO_bytecode'] = module - finally: - imp.release_lock() - else: - Log.warn('ObjBase','attempt to supply code denied: ',name,'from',str(sourceaddr)) - raise PyroError('attempt to supply code denied') - - # remote code retrieve support (client retrieves from server): - def remote_retrieve_code(self, name): - # XXX codeValidator: can we somehow get the client's address it is sent to? - # XXX this code is ugly. And duplicated in protocol.py remoteInvocation. - if Pyro.config.PYRO_MOBILE_CODE and self.codeValidator(name,None,None): - Log.msg("ObjBase","supplying code: ",name) - try: - importmodule=new.module("pyro-server-import") - try: - exec "import " + name in importmodule.__dict__ - except ImportError: - Log.error("ObjBase","Client wanted a non-existing module:", name) - raise PyroError("Client wanted a non-existing module", name) - m=eval("importmodule."+name) - # try to load the module's compiled source, or the real .py source if that fails. - # note that the source code (.py) is opened with universal newline mode - (filebase,ext)=os.path.splitext(m.__file__) - if ext.startswith(".PY"): - exts = ( (".PYO","rb"), (".PYC","rb"), (".PY","rU") ) # uppercase - else: - exts = ( (".pyo","rb"), (".pyc","rb"), (".py","rU") ) # lowercase - for ext,mode in exts: - try: - m=open(filebase+ext, mode).read() - return m # supply the module to the client! - except: - pass - Log.error("ObjBase","cannot read module source code for module:", name) - raise PyroError("cannot read module source code") - finally: - del importmodule - else: - Log.error("ObjBase","attempt to retrieve code denied:", name) - raise PyroError("attempt to retrieve code denied") - - -class SynchronizedObjBase(ObjBase): - def __init__(self): - ObjBase.__init__(self) - self.synlock=Pyro.util.getLockObject() - def Pyro_dyncall(self, method, flags, args): - with self.synlock: - return ObjBase.Pyro_dyncall(self, method,flags,args) - - -# Use this class instead if you're using callback objects and you -# want to see local exceptions. (otherwise they go back to the calling server...) -class CallbackObjBase(ObjBase): - def __init__(self): - ObjBase.__init__(self) - def Pyro_dyncall(self, method, flags, args): - try: - return ObjBase.Pyro_dyncall(self,method,flags,args) - except Exception,x: - # catch all errors - Log.warn('CallbackObjBase','Exception in callback object: ',x) - raise PyroExceptionCapsule(x,str(x)) - - -############################################################################# -# -# PyroURI - Pyro Universal Resource Identifier -# -# This class represents a Pyro URI (which consists of four parts, -# a protocol identifier, an IP address, a portnumber, and an object ID. -# -# The URI can be converted to a string representation (str converter). -# The URI can also be read back from such a string (reinitFromString). -# The URI can be initialised from its parts (init). -# The URI can be initialised from a string directly, if the init -# code detects a ':' and '/' in the host argument (which is then -# assumed to be a string URI, not a host name/ IP address). -# -############################################################################# - -class PyroURI(object): - def __init__(self,host,objectID=0,port=0,prtcol='PYRO'): - # if the 'host' arg is a PyroURI, copy contents - if isinstance(host, PyroURI): - self.init(host.address, host.objectID, host.port, host.protocol) - else: - # If the 'host' arg contains '://', assume it's an URI string. - if host.find('://')>0: - self.reinitFromString(host) - else: - if not objectID: - raise URIError('invalid URI format') - self.init(host, objectID, port, prtcol) - def __str__(self): - return self.protocol+'://'+self.address+':'+str(self.port)+'/'+self.objectID - def __repr__(self): - return '' - def __hash__(self): - # XXX this is handy but not safe. If the URI changes, the object will be in the wrong hash bucket. - return hash(str(self)) - def __cmp__(self, o): - return cmp(str(self), str(o)) - def clone(self): - return PyroURI(self) - def init(self,host,objectID,port=0,prtcol='PYRO'): - if '/' in host: - raise URIError('malformed hostname') - if Pyro.config.PYRO_DNS_URI: - self.address = host - else: - self.address=Pyro.protocol.getIPAddress(host) - if not self.address: - raise URIError('unknown host') - if port: - if type(port)==type(1): - self.port=port - else: - raise TypeError("port must be integer") - else: - self.port=Pyro.config.PYRO_PORT - self.protocol=prtcol - self.objectID=objectID - def reinitFromString(self,arg): - if arg.startswith('PYROLOC') or arg.startswith('PYRONAME'): - uri=processStringURI(arg) - self.init(uri.address,uri.objectID,uri.port,uri.protocol) - return - x=re.match(r'(?P[^\s:/]+)://(?P[^\s:]+):?(?P\d+)?/(?P\S*)',arg) - if x: - port=None - if x.group('port'): - port=int(x.group('port')) - self.init(x.group('hostname'), x.group('id'), port, x.group('protocol')) - return - Log.error('PyroURI','invalid URI format passed: '+arg) - raise URIError('invalid URI format') - def getProxy(self): - return DynamicProxy(self) - def getAttrProxy(self): - return DynamicProxyWithAttrs(self) - - -# -# This method takes a string representation of a Pyro URI -# and parses it. If it's a meta-protocol URI such as -# PYRONAME://.... it will do what is needed to make -# a regular PYRO:// URI out of it (resolve names etc). -# -def processStringURI(URI): - # PYRONAME(SSL)://[hostname[:port]/]objectname - x=re.match(r'(?PPYRONAME|PYRONAMESSL)://(((?P[^\s:]+):(?P\d+)/)|((?P[^\s:]+)/))?(?P\S*)',URI) - if x: - protocol=x.group('protocol') - if protocol=="PYRONAMESSL": - raise ProtocolError("NOT SUPPORTED YET: "+protocol) # XXX obviously, this should be implemented - hostname=x.group('hostname') or x.group('onlyhostname') - port=x.group('port') - name=x.group('name') - import Pyro.naming - loc=Pyro.naming.NameServerLocator() - if port: - port=int(port) - NS=loc.getNS(host=hostname,port=port) - return NS.resolve(name) - # PYROLOC(SSL)://hostname[:port]/objectname - x=re.match(r'(?PPYROLOC|PYROLOCSSL)://(?P[^\s:]+):?(?P\d+)?/(?P\S*)',URI) - if x: - protocol=x.group('protocol') - hostname=x.group('hostname') - port=x.group('port') - if port: - port=int(port) - else: - port=0 - name=x.group('name') - return PyroURI(hostname,name,port,protocol) - if URI.startswith('PYROLOC') or URI.startswith('PYRONAME'): - # hmm should have matched above. Likely invalid. - raise URIError('invalid URI format') - # It's not a meta-protocol such as PYROLOC or PYRONAME, - # let the normal Pyro URI deal with it. - # (it can deal with regular PYRO: and PYROSSL: protocols) - return PyroURI(URI) - - -############################################################################# -# -# DynamicProxy - dynamic Pyro proxy -# -# Can be used by clients to invoke objects for which they have no -# precompiled proxy. -# -############################################################################# - -def getProxyForURI(URI): - return DynamicProxy(URI) -def getAttrProxyForURI(URI): - return DynamicProxyWithAttrs(URI) - -class _RemoteMethod(object): - # method call abstraction, adapted from Python's xmlrpclib - # it would be rather easy to add nested method calls, but - # that is not compatible with the way that Pyro's method - # calls are defined to work ( no nested calls ) - def __init__(self, send, name): - self.__send = send - self.__name = name - def __call__(self, *args, **kwargs): - return self.__send(self.__name, args, kwargs) - -class DynamicProxy(object): - def __init__(self, URI): - _checkInit() # init required - if type(URI) in (StringType,UnicodeType): - URI=processStringURI(URI) - self.URI = URI - self.objectID = URI.objectID - # Delay adapter binding to enable transporting of proxies. - # We just create an adapter, and don't connect it... - self.adapter = Pyro.protocol.getProtocolAdapter(self.URI.protocol) - # ---- don't forget to register local vars with DynamicProxyWithAttrs, see below - def __del__(self): - try: - self.adapter.release(nolog=1) - except (AttributeError, RuntimeError): - pass - def _setIdentification(self, ident): - self.adapter.setIdentification(ident) - def _setNewConnectionValidator(self, validator): - self.adapter.setNewConnectionValidator(validator) - def _setOneway(self, methods): - if type(methods) not in (type([]), type((0,))): - methods=(methods,) - self.adapter.setOneway(methods) - def _setTimeout(self,timeout): - self.adapter.setTimeout(timeout) - def _transferThread(self, newOwnerThread=None): - pass # dummy function to retain API compatibility with Pyro 3.7 - def _release(self): - if self.adapter: - self.adapter.release() - def _local(self): - return self.URI._local() - def _islocal(self): - return self.URI._islocal() - def __copy__(self): # create copy of current proxy object - proxyCopy = DynamicProxy(self.URI) - proxyCopy.adapter.setIdentification(self.adapter.getIdentification(), munge=False) # copy identification info - proxyCopy._setTimeout(self.adapter.timeout) - proxyCopy._setOneway(self.adapter.onewayMethods) - proxyCopy._setNewConnectionValidator(self.adapter.getNewConnectionValidator()) - return proxyCopy - def __deepcopy__(self, arg): - raise PyroError("cannot deepcopy a proxy") - def __getattr__(self, name): - if name in ("__getnewargs__","__getinitargs__"): # allows it to be safely pickled - raise AttributeError() - return _RemoteMethod(self._invokePYRO, name) - def __repr__(self): - return "<"+self.__class__.__name__+" for "+str(self.URI)+">" - def __str__(self): - return repr(self) - def __hash__(self): - # makes it possible to use this class as a key in a dict - return hash(self.objectID) - def __eq__(self,other): - # makes it possible to compare two proxies using objectID - return hasattr(other,"objectID") and self.objectID==other.objectID - def __ne__(self,other): - # makes it possible to compare two proxies using objectID - return not hasattr(other,"objectID") or self.objectID!=other.objectID - def __nonzero__(self): - return 1 - def __coerce__(self,other): - # makes it possible to compare two proxies using objectID (cmp) - if hasattr(other,"objectID"): - return (self.objectID, other.objectID) - return None - - def _invokePYRO(self, name, vargs, kargs): - if not self.adapter.connected(): - # rebind here, don't do it from inside the remoteInvocation because deadlock will occur - self.adapter.bindToURI(self.URI) - return self.adapter.remoteInvocation(name, Pyro.constants.RIF_VarargsAndKeywords, vargs, kargs) - - # Pickling support, otherwise pickle uses __getattr__: - def __getstate__(self): - # for pickling, return a non-connected copy of ourselves: - cpy = self.__copy__() - cpy._release() - return cpy.__dict__ - def __setstate__(self, args): - # for pickling, to restore the pickled state - self.__dict__.update(args) - - -class DynamicProxyWithAttrs(DynamicProxy): - _local_attrs = ("_local_attrs","URI", "objectID", "adapter", "_attr_cache") - def __init__(self, URI): - self._attr_cache = {} - DynamicProxy.__init__(self, URI) - def _r_ga(self, attr, value=0): - if value: - return _RemoteMethod(self._invokePYRO, "_r_ga") (attr) # getattr - else: - return _RemoteMethod(self._invokePYRO, "_r_ha") (attr) # hasattr - def findattr(self, attr): - if attr in self._attr_cache.keys(): - return self._attr_cache[attr] - # look it up and cache the value - self._attr_cache[attr] = self._r_ga(attr) - return self._attr_cache[attr] - def __copy__(self): # create copy of current proxy object - return DynamicProxyWithAttrs(self.URI) - def __setattr__(self, attr, value): - if attr in self._local_attrs: - self.__dict__[attr]=value - else: - result = self.findattr(attr) - if result==2: # attribute - return _RemoteMethod(self._invokePYRO, "_r_sa") (attr,value) - else: - raise AttributeError('not an attribute') - def __getattr__(self, attr): - # allows it to be safely pickled - if attr not in ("__getnewargs__","__getinitargs__", "__hash__","__eq__","__ne__") and attr not in self._local_attrs: - result=self.findattr(attr) - if result==1: # method - return _RemoteMethod(self._invokePYRO, attr) - elif result: - return self._r_ga(attr, 1) - raise AttributeError - - -############################################################################# -# -# Daemon - server-side Pyro daemon -# -# Accepts and dispatches incoming Pyro method calls. -# -############################################################################# - -# The pyro object that represents the daemon. -# The daemon is not directly remotely accessible, for security reasons. -class DaemonServant(ObjBase): - def __init__(self, daemon): - ObjBase.__init__(self) - self.daemon=weakref.proxy(daemon) - def getRegistered(self): - return self.daemon.getRegistered() - def ResolvePYROLOC(self, name): - return self.daemon.ResolvePYROLOC(name) - -# The daemon itself: -class Daemon(Pyro.protocol.TCPServer, ObjBase): - def __init__(self,prtcol='PYRO',host=None,port=0,norange=0,publishhost=None): - ObjBase.__init__(self) - self.NameServer = None - self.connections=[] - _checkInit("server") # init required - self.setGUID(Pyro.constants.INTERNAL_DAEMON_GUID) - self.implementations={Pyro.constants.INTERNAL_DAEMON_GUID:(DaemonServant(self),'__PYRO_Internal_Daemon')} - self.persistentConnectedObjs=[] # guids - self.transientsCleanupAge=0 - self.transientsMutex=Pyro.util.getLockObject() - self.nscallMutex=Pyro.util.getLockObject() - if host is None: - host=Pyro.config.PYRO_HOST - if publishhost is None: - publishhost=Pyro.config.PYRO_PUBLISHHOST - - # Determine range scanning or random port allocation - if norange: - # Fixed or random port allocation - # If port is zero, OS will randomly assign, otherwise, - # attempt to use the provided port value - self.port = port - portrange = 1 - else: - # Scanning port allocation - if port: - self.port = port - else: - self.port = Pyro.config.PYRO_PORT - portrange=Pyro.config.PYRO_PORT_RANGE - - if not publishhost: - publishhost=host - errormsg='' - for i in range(portrange): - try: - Pyro.protocol.TCPServer.__init__(self, self.port, host, Pyro.config.PYRO_MULTITHREADED,prtcol) - if not self.port: - # If we bound to an OS provided port, report it - self.port = self.sock.getsockname()[1] - self.hostname = publishhost or Pyro.protocol.getHostname() - self.protocol = prtcol - self.adapter = Pyro.protocol.getProtocolAdapter(prtcol) - self.validateHostnameAndIP() # ignore any result message... it's in the log already. - return - except ProtocolError,msg: - errormsg=msg - self.port+=1 - Log.error('Daemon','Couldn\'t start Pyro daemon: ' +str(errormsg)) - raise DaemonError('Couldn\'t start Pyro daemon: ' +str(errormsg)) - - # to be called to stop all connections and shut down. - def shutdown(self, disconnect=False): - Pyro.protocol.TCPServer.shutdown(self) - if disconnect: - self.__disconnectObjects() - def __disconnectObjects(self): - # server shutting down, unregister all known objects in the NS - if self.NameServer and Pyro and Pyro.constants: - with self.nscallMutex: - if Pyro.constants.INTERNAL_DAEMON_GUID in self.implementations: - del self.implementations[Pyro.constants.INTERNAL_DAEMON_GUID] - if self.implementations: - Log.warn('Daemon','Shutting down but there are still',len(self.implementations),'objects connected - disconnecting them') - for guid in self.implementations.keys(): - if guid not in self.persistentConnectedObjs: - (obj,name)=self.implementations[guid] - if name: - try: - self.NameServer.unregister(name) - except Exception,x: - Log.warn('Daemon','Error while unregistering object during shutdown:',x) - self.implementations={} - - def __del__(self): - self.__disconnectObjects() # unregister objects - try: - del self.adapter - Pyro.protocol.TCPServer.__del__(self) - except (AttributeError, RuntimeError): - pass - - def __str__(self): - return '' - def __getstate__(self): - raise PicklingError('no access to the daemon') - - def validateHostnameAndIP(self): - # Checks if hostname is sensible. Returns None if it is, otherwise a message - # telling what's wrong if it isn't too serious. If things are really bad, - # expect an exception to be raised. Things are logged too. - if not self.hostname: - Log.error("Daemon","no hostname known") - raise socket.error("no hostname known for daemon") - if self.hostname!="localhost": - ip = Pyro.protocol.getIPAddress(self.hostname) - if ip is None: - Log.error("Daemon","no IP address known") - raise socket.error("no IP address known for daemon") - if not ip.startswith("127.0."): - return None # this is good! - # 127.0.x.x or 'localhost' is a warning situation! - msg="daemon bound on hostname that resolves to loopback address 127.0.x.x" - Log.warn("Daemon",msg) - Log.warn("Daemon","hostname="+self.hostname) - return msg - - def useNameServer(self,NS): - self.NameServer=NS - def getNameServer(self): - return self.NameServer - def setTimeout(self, timeout): - self.adapter.setTimeout(timeout) - def setAllowedIdentifications(self, ids): - self.getNewConnectionValidator().setAllowedIdentifications(ids) - def setTransientsCleanupAge(self, secs): - self.transientsCleanupAge=secs - if self.threaded: - Log.msg('Daemon','creating Grim Reaper thread for transients, timeout=',secs) - reaper=threading.Thread(target=self._grimReaper) - reaper.setDaemon(1) # thread must exit at program termination. - reaper.start() - def _grimReaper(self): - # this runs in a thread. - while self.transientsCleanupAge>0: - time.sleep(self.transientsCleanupAge/5) - self.reapUnusedTransients() - - def getProxyForObj(self, obj): - return DynamicProxy( PyroURI(self.hostname, - obj.GUID(), prtcol=self.protocol, port=self.port) ) - def getAttrProxyForObj(self, obj): - return DynamicProxyWithAttrs( PyroURI(self.hostname, - obj.GUID(), prtcol=self.protocol, port=self.port) ) - - def connectPersistent(self, obj, name=None): - # when a persistent entry is found in the NS, that URI is - # used instead of the supplied one, if the address matches. - if name and self.NameServer: - with self.nscallMutex: - try: - newURI = PyroURI(self.hostname, obj.GUID(), prtcol=self.protocol, port=self.port) - URI=self.NameServer.resolve(name) - if (URI.protocol,URI.address,URI.port)==(newURI.protocol,newURI.address,newURI.port): - # reuse the previous object ID - obj.setGUID(URI.objectID) - # enter the (object,name) in the known impl. dictionary - self.implementations[obj.GUID()]=(obj,name) - self.persistentConnectedObjs.append(obj.GUID()) - obj.setPyroDaemon(self) - return URI - else: - # name exists, but address etc. is wrong. Remove it. - # then continue so it wil be re-registered. - try: self.NameServer.unregister(name) - except NamingError: pass - except NamingError: - pass - # Register normally. - self.persistentConnectedObjs.append(obj.GUID()) - return self.connect(obj, name) - - def connect(self, obj, name=None): - URI = PyroURI(self.hostname, obj.GUID(), prtcol=self.protocol, port=self.port) - # if not transient, register the object with the NS - if name: - with self.nscallMutex: - if self.NameServer: - self.NameServer.register(name, URI) - else: - Log.warn('Daemon','connecting object without name server specified:',name) - # enter the (object,name) in the known implementations dictionary - self.implementations[obj.GUID()]=(obj,name) - obj.setPyroDaemon(self) - return URI - - def disconnect(self,obj): # obj can be either the object that was registered, or its uid - try: - if isinstance(obj,Pyro.core.ObjBase): - obj_uid=obj.GUID() - else: - obj_uid=str(obj) - if obj_uid==Pyro.constants.INTERNAL_DAEMON_GUID: - return # never allow to remove ourselves from the registry - if self.NameServer and self.implementations[obj_uid][1]: - with self.nscallMutex: - # only unregister with NS if it had a name (was not transient) - self.NameServer.unregister(self.implementations[obj_uid][1]) - del self.implementations[obj_uid] - if obj_uid in self.persistentConnectedObjs: - self.persistentConnectedObjs.remove(obj_uid) - # XXX Clean up connections/threads to this object? - # Can't be done because thread/socket is not associated with single object - finally: - if isinstance(obj,Pyro.core.ObjBase): - obj.setPyroDaemon(None) - - def getRegistered(self): - r={} - for guid in self.implementations.keys(): - r[guid]=self.implementations[guid][1] # keep only the names - return r - - def handleInvocation(self, conn): # overridden from TCPServer - # called in both single- and multithreaded mode - self.getLocalStorage().caller=conn - self.getAdapter().handleInvocation(self, conn) - self.reapUnusedTransients() - - def reapUnusedTransients(self): - if not self.transientsCleanupAge: return - now=time.time() - with self.transientsMutex: - for (obj,name) in self.implementations.values()[:]: # use copy of list - if not name: - # object is transient, reap it if timeout requires so. - if (now-obj.lastUsed)>self.transientsCleanupAge: - self.disconnect(obj) - obj._gotReaped() - - def handleError(self,conn,onewaycall=False): # overridden from TCPServer - try: - (exc_type, exc_value, exc_trb) = sys.exc_info() - if exc_type==ProtocolError: - # Problem with the communication protocol, shut down the connection - # XXX is shutting down what we want??? - Log.error('Daemon','protocol error occured:',exc_value) - Log.error('Daemon','Due to network error: shutting down connection with',conn) - self.removeConnection(conn) - else: - exclist = Pyro.util.formatTraceback(exc_type, exc_value, exc_trb) - out =''.join(exclist) - Log.warn('Daemon', 'Exception during processing of request from', - conn,' type',exc_type, - '\n--- traceback of this exception follows:\n', - out,'\n--- end of traceback') - if exc_type==PyroExceptionCapsule: - sys.stdout.flush() - # This is a capsuled exception, used with callback objects. - # That means we are actually the daemon on the client. - # Return the error to the other side and raise exception locally once more. - # (with a normal exception, it is not raised locally again!) - # only send the exception object if it's not a oneway call. - if not onewaycall: - self.adapter.returnException(conn,exc_value.excObj,0,exclist) # don't shutdown - exc_value.raiseEx() - else: - # normal exception, only return exception object if it's not a oneway call - if not onewaycall: - self.adapter.returnException(conn,exc_value,0,exclist) # don't shutdown connection - - finally: - # clean up circular references to traceback info to allow proper GC - del exc_type, exc_value, exc_trb - - def getAdapter(self): - # overridden from TCPServer - return self.adapter - - def getLocalObject(self, guid): - # return a local object registered with the given guid - return self.implementations[guid][0] - def getLocalObjectForProxy(self, proxy): - # return a local object registered with the guid to which the given proxy points - return self.implementations[proxy.objectID][0] - - def ResolvePYROLOC(self, name): - # this gets called from the protocol adapter when - # it wants the daemon to resolve a local object name (PYROLOC: protocol) - Log.msg('Daemon','resolving PYROLOC name: ',name) - for o in self.implementations.keys(): - if self.implementations[o][1]==name: - return o - raise NamingError('no object found by this name',name) - - -############################################################################# -# -# Client/Server Init code -# -############################################################################# - -# Has init been performed already? -_init_server_done=0 -_init_client_done=0 -_init_generic_done=0 - -def _initGeneric_pre(): - global _init_generic_done - if _init_generic_done: - return - if Pyro.config.PYRO_TRACELEVEL == 0: return - try: - out='\n'+'-'*60+' NEW SESSION\n'+time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))+ \ - ' Pyro Initializing, version '+Pyro.constants.VERSION+'\n' - Log.raw(out) - except IOError,e: - sys.stderr.write('PYRO: Can\'t write the tracefile '+Pyro.config.PYRO_LOGFILE+'\n'+str(e)) - -def _initGeneric_post(): - global _init_generic_done - setattr(Pyro.config, Pyro.constants.CFGITEM_PYRO_INITIALIZED,1) - if Pyro.config.PYRO_TRACELEVEL == 0: return - try: - if not _init_generic_done: - out='Configuration settings are as follows:\n' - for item in dir(Pyro.config): - if item[0:4] =='PYRO': - out+=item+' = '+str(Pyro.config.__dict__[item])+'\n' - Log.raw(out) - Log.raw('Init done.\n'+'-'*70+'\n') - except IOError: - pass - _init_generic_done=1 - - -def initClient(banner=0): - global _init_client_done - if _init_client_done: return - _initGeneric_pre() - if Pyro.config.PYRO_TRACELEVEL >0: Log.raw('This is initClient.\n') - Pyro.config.finalizeConfig_Client() - _initGeneric_post() - if banner: - print 'Pyro Client Initialized. Using Pyro V'+Pyro.constants.VERSION - _init_client_done=1 - -def initServer(banner=0, storageCheck=1): - global _init_server_done - if _init_server_done: return - _initGeneric_pre() - if Pyro.config.PYRO_TRACELEVEL >0: Log.raw('This is initServer.\n') - Pyro.config.finalizeConfig_Server(storageCheck=storageCheck) - _initGeneric_post() - if banner: - print 'Pyro Server Initialized. Using Pyro V'+Pyro.constants.VERSION - _init_server_done=1 - - -if __name__=="__main__": - print "Pyro version:",Pyro.constants.VERSION diff --git a/lib/Pyro/errors.py b/lib/Pyro/errors.py deleted file mode 100644 index 323668d5a0a..00000000000 --- a/lib/Pyro/errors.py +++ /dev/null @@ -1,82 +0,0 @@ -############################################################################# -# -# Pyro Exception Types -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -############################################################################# -# PyroError is the Pyro exception type which is used for problems WITHIN Pyro. -# User code should NOT use it!! -# -# NOTE: Any exception occuring in the user code on the server will be catched -# and transported to the client, where it is raised just as if it occured -# locally. The occurrence is logged in the server side Pyro log. -# Pyro will use the [Remote]PyroError exceptions and their subtypes to -# indicate that an internal problem occured. -############################################################################# - -class PyroError(Exception): pass # internal - -class URIError(PyroError): pass # URI probs -class DaemonError(PyroError): pass # daemon probs -class ProtocolError(PyroError): pass # protocol adapter -class ConnectionClosedError(ProtocolError): pass # connection in adapter is closed -class ConnectionDeniedError(ProtocolError): pass # server refused connection -class TimeoutError(ConnectionClosedError): pass # communication timeout -class NamingError(PyroError): pass # name server -class NoModuleError(PyroError): pass # no module found for incoming obj - -# do NOT use the following yourself: -class _InternalNoModuleError(PyroError): - def __init__(self, modulename=None, fromlist=None, *args): - # note: called without args on Python 2.5+, args will be set by __setstate__ - self.modulename=modulename - self.fromlist=fromlist - PyroError.__init__(* (self,)+args) - def __getstate__(self): - return { "modulename": self.modulename, "fromlist": self.fromlist } - def __setstate__(self, state): - self.modulename=state["modulename"] - self.fromlist=state["fromlist"] - - -############################################################################# -# -# PyroExceptionCapsule - Exception encapsulation. -# -# This class represents a Pyro exception which can be transported -# across the network, and raised on the other side (by invoking raiseEx). -# NOTE: the 'real' exception class must be 'known' on the other side! -# NOTE2: this class is adapted from exceptions.Exception. -# NOTE3: PyroError IS USED FOR ACTUAL PYRO ERRORS. PyroExceptionCapsule -# IS ONLY TO BE USED TO TRANSPORT AN EXCEPTION ACROSS THE NETWORK. -# NOTE4: It sets a special attribute on the exception that is raised -# (constants.TRACEBACK_ATTRIBUTE), this is the *remote* traceback -# NOTE5: ---> this class is intentionally not subclassed from Exception, -# and also not from object. -# Pyro's exception handling depends on this! -# -############################################################################# - -import Pyro.constants - -class PyroExceptionCapsule: # don't make this a new style class - def __init__(self,excObj,args=None): - self.excObj = excObj - self.args=args # if specified, this is the remote traceback info - def raiseEx(self): - setattr(self.excObj,Pyro.constants.TRACEBACK_ATTRIBUTE,self.args) - raise self.excObj - def __str__(self): - s=self.excObj.__class__.__name__ - if not self.args: - return s - elif len(self.args) == 1: - return s+': '+str(self.args[0]) - else: - return s+': '+str(self.args) - def __getitem__(self, i): - return self.args[i] diff --git a/lib/Pyro/ext/BasicNTService.py b/lib/Pyro/ext/BasicNTService.py deleted file mode 100644 index 327e0b678da..00000000000 --- a/lib/Pyro/ext/BasicNTService.py +++ /dev/null @@ -1,196 +0,0 @@ -############################################################################# -# -# An NT service that runs the Pyro Name Server -# Author: Syver Enstad; syver-en@online.no -# Bugfix for recent win32 builds: David Rushby; woodsplitter@rocketmail.com -# -# This is part of "Pyro" - Python Remote Objects -# Which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -import sys -import win32serviceutil -import threading -import win32service -import win32api -import win32con - - -class BasicNTService(win32serviceutil.ServiceFramework, object): - """ Abstract base to help out with building NT services - in Python with the win32all(by Mark Hammond) support for - python nt services. - - Remember to set the two following class attributes - to something sensible in your subclass - _svc_name_ = 'PyroNS' - _svc_display_name_ = 'Pyro Naming Service NT service' - - The following are optional - _svc_deps_: This should be set to the list of service names - That need to be started before this one. - _exe_name_: This should be set to a service .EXE if you're not - going to use PythonService.exe - _svc_description_ : This is the descriptive string that you find - in the services applet - - To register the service with the SCM the easiest way is to include the - following at the bottom of the file where your subclass is defined. - if __name__ == '__main__': - TheClassYouDerivedFromBasicNTService.HandleCommandLine() - - """ - def __init__(self, args): - _redirectSystemStreamsIfNecessary() - - win32serviceutil.ServiceFramework.__init__(self, args) - self._stopEvent = threading.Event() - - def SvcStop(self): - """ Template method from win32serviceutil.ServiceFramework""" - # first tell SCM that we have started the stopping process - self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) - self._stopEvent.set() - - def _shouldStop(self): - return self._stopEvent.isSet() - - def _doRun(self): - raise NotImplementedError - - def _doStop(self): - raise NotImplementedError - - def SvcDoRun(self): - """ part of Template method SvcRun - from win32serviceutil.ServiceFramework""" - self.logStarted() - self._doRun() - self._stopEvent.wait() - self._doStop() - self.logTermination() - return 0 - - def logTermination(self): - import servicemanager - servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE, - servicemanager.PYS_SERVICE_STOPPED, - (self._svc_name_, "")) - - def logStarted(self): - import servicemanager - servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE, - servicemanager.PYS_SERVICE_STARTED, - (self._svc_name_, '')) - - def CustomOptionHandler(cls, opts): - #out=open("c:\\log.txt","w") - print "Installing the Pyro %s" % cls._svc_name_ - args = raw_input("Enter command line arguments for %s: " % cls._svc_name_) - try: - createRegistryParameters(cls._svc_name_, args.strip()) - except Exception,x: - print "Error occured when setting command line args in the registry: ",x - try: - cls._svc_description_ - except LookupError: - return - - key = win32api.RegCreateKey(win32con.HKEY_LOCAL_MACHINE, - "System\\CurrentControlSet\\Services\\%s" % cls._svc_name_) - try: - win32api.RegSetValueEx(key, "Description", 0, win32con.REG_SZ, cls._svc_description_); - finally: - win32api.RegCloseKey(key) - CustomOptionHandler = classmethod(CustomOptionHandler) - - - def HandleCommandLine(cls): - if win32serviceutil.HandleCommandLine(cls, customOptionHandler=cls.CustomOptionHandler) != 0: - return # some error occured - if sys.argv[1] in ("install", "update"): - print "\nYou can configure the command line arguments in the Registry." - print "The key is: HKLM\\System\\CurrentControlSet\\Services\\%s" % cls._svc_name_ - print "The value under that key is: ", pyroArgsRegkeyName - args=getRegistryParameters(cls._svc_name_) - if args: - print "(it is currently set to: '%s')" % args - else: - print "(it is currently not set)" - print - HandleCommandLine = classmethod(HandleCommandLine) - - - -pyroArgsRegkeyName = "PyroServiceArguments" - - -def getRegistryParameters(servicename): - key=win32api.RegOpenKey(win32con.HKEY_LOCAL_MACHINE, "System\\CurrentControlSet\\Services\\"+servicename) - try: - try: - (commandLine, regtype) = win32api.RegQueryValueEx(key,pyroArgsRegkeyName) - return commandLine - except: - pass - finally: - key.Close() - - createRegistryParameters(servicename, pyroArgsRegkeyName) - return "" - - -def createRegistryParameters(servicename, parameters): - newkey=win32api.RegOpenKeyEx(win32con.HKEY_LOCAL_MACHINE, "System\\CurrentControlSet\\Services\\"+servicename,0,win32con.KEY_ALL_ACCESS) - try: - win32api.RegSetValueEx(newkey, pyroArgsRegkeyName, 0, win32con.REG_SZ, parameters) - finally: - newkey.Close() - - -def _redirectSystemStreamsIfNecessary(): - # Python programs running as Windows NT services must not send output to - # the default sys.stdout or sys.stderr streams, because those streams are - # not fully functional in the NT service execution environment. Sending - # output to them will eventually (but not immediately) cause an IOError - # ("Bad file descriptor"), which can be quite mystifying to the - # uninitiated. This problem can be overcome by replacing the default - # system streams with a stream that discards any data passed to it (like - # redirection to /dev/null on Unix). - # - # However, the pywin32 service framework supports a debug mode, under which - # the streams are fully functional and should not be redirected. - shouldRedirect = True - try: - import servicemanager - except ImportError: - # If we can't even 'import servicemanager', we're obviously not running - # as a service, so the streams shouldn't be redirected. - shouldRedirect = False - else: - # Unlike previous builds, pywin32 builds >= 200 allow the - # servicemanager module to be imported even in a program that isn't - # running as a service. In such a situation, it would not be desirable - # to redirect the system streams. - # - # However, it was not until pywin32 build 203 that a 'RunningAsService' - # predicate was added to allow client code to determine whether it's - # running as a service. - # - # This program logic redirects only when necessary if using any build - # of pywin32 except 200-202. With 200-202, the redirection is a bit - # more conservative than is strictly necessary. - if ( - servicemanager.Debugging() - or ( - hasattr(servicemanager, 'RunningAsService') - and not servicemanager.RunningAsService() - ) - ): - shouldRedirect = False - - if shouldRedirect: - sys.stdout = sys.stderr = open('nul', 'w') - - return shouldRedirect diff --git a/lib/Pyro/ext/ES_NtService.py b/lib/Pyro/ext/ES_NtService.py deleted file mode 100644 index b21cefdc996..00000000000 --- a/lib/Pyro/ext/ES_NtService.py +++ /dev/null @@ -1,112 +0,0 @@ -############################################################################# -# -# An NT service that runs the Pyro Event Service -# Author: Syver Enstad syver-en@online.no -# -# This is part of "Pyro" - Python Remote Objects -# Which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -import win32serviceutil -import threading -import win32service -import win32api -from BasicNTService import BasicNTService, getRegistryParameters - - -def setConfig(): - Pyro.config.PYRO_TRACELEVEL=3 - Pyro.config.PYRO_STORAGE = os.path.splitdrive(win32api.GetSystemDirectory())[0]+os.sep - Pyro.config.PYRO_LOGFILE = "Pyro_ES_svc.log" - - -import os,sys -import Pyro.util -setConfig() -Log=Pyro.util.Log -from Pyro.EventService import Server - - -class PyroESThread(threading.Thread): - """ The Pyro Event Service will run in this thread - """ - def __init__(self, args, stopcallback): - threading.Thread.__init__(self) - self._args = list(args) - self._stopcallback = stopcallback - - def run(self): - self.startPyroES() - self._stopcallback() - - def startPyroES(self): - Log.msg("PyroES_svc","Pyro ES service is starting, arguments=",self._args) - """ ripped out of Pyro.EventService.Server and slightly changed to - accomodate not using sys.argv, but self._args instead - """ - - try: - Args = Pyro.util.ArgParser() - Args.parse(self._args,'hn:p:i:') - if Args.hasOpt('h'): - Log.error("PyroES_svc",""" -Usage: es [-h] [-n hostname] [-p port] [-i identification] - where -p = ES server port - -n = non-default hostname to bind on - -i = the required authentication ID for ES clients, - also used to connect to other Pyro services - -h = print this help -""") - raise SystemExit - host = Args.getOpt('n','') - port = Args.getOpt('p',None) - ident = Args.getOpt('i',None) - if port: - port=int(port) - if Args.ignored: - Log.warn("PyroES_svc",'Ignored options:',Args.ignored) - if Args.args: - Log.warn("PyroES_svc",'Ignored arguments:',Args.args) - - Log.msg("PyroES_scv", "Starting the Event Server.") - self.starter=Server.EventServiceStarter(identification=ident) - self.starter.start(host,port) - - except Exception,x : - Log.error("PyroES_scv","COULD NOT START!!!",x) - raise SystemExit - - def shutdown(self): - self.starter.running=0 - - -class PyroES_NTService(BasicNTService): - _svc_name_ = 'PyroES' - _svc_display_name_ = "Pyro Event Service" - _svc_description_ = "Provides event topics and publish/subscribe communication for Pyro" - def __init__(self, args): - super(PyroES_NTService, self).__init__(args) - setConfig() - try: - args = getRegistryParameters(self._svc_name_).split() - except Exception,x: - Log.error("PyroES_svc","PROBLEM GETTING ARGS FROM REGISTRY:",x) - self._esThread = PyroESThread(args, self.SvcStop) - - def _doRun(self): - """ Overriden """ - self._esThread.start() - - def _doStop(self): - """ Overridden """ - self._esThread.shutdown() - self._esThread.join() - - def SvcStop(self): - """Overriden """ - super(PyroES_NTService, self).SvcStop() - - -if __name__ == '__main__': - PyroES_NTService.HandleCommandLine() diff --git a/lib/Pyro/ext/NS_NtService.py b/lib/Pyro/ext/NS_NtService.py deleted file mode 100644 index a0af0f06690..00000000000 --- a/lib/Pyro/ext/NS_NtService.py +++ /dev/null @@ -1,200 +0,0 @@ -############################################################################# -# -# An NT service that runs the Pyro Name Server -# Author: Syver Enstad syver-en@online.no -# -# This is part of "Pyro" - Python Remote Objects -# Which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -import win32serviceutil -import threading -import win32service -import win32api -from BasicNTService import BasicNTService, getRegistryParameters - - -def setConfig(): - Pyro.config.PYRO_TRACELEVEL=3 - Pyro.config.PYRO_STORAGE = os.path.splitdrive(win32api.GetSystemDirectory())[0]+os.sep - Pyro.config.PYRO_LOGFILE = "Pyro_NS_svc.log" - Pyro.config.PYRO_NS_URIFILE = os.path.join(Pyro.config.PYRO_STORAGE, "Pyro_NS_URI.txt") - -import os,sys -import Pyro.util -setConfig() -Log=Pyro.util.Log -import Pyro.core -import Pyro.constants -import Pyro.naming -from Pyro.naming import NameServer, PersistentNameServer, BroadcastServer, bcRequestHandler - -BcServerObject = None - -def startServer(hstn='', nsport=0, bcport=0, keep=0, persistent=0, dbdir=None, Guards=(None,None)): - global BcServerObject - if not nsport: - nsport=Pyro.config.PYRO_NS_PORT - if not bcport: - bcport=Pyro.config.PYRO_NS_BC_PORT - Pyro.core.initServer() - PyroDaemon = Pyro.core.Daemon(host=hstn, port=nsport,norange=1) - if Guards[0]: - PyroDaemon.setNewConnectionValidator(Guards[0]) - if persistent: - ns=PersistentNameServer(dbdir) - PyroDaemon.useNameServer(ns) - NS_URI=PyroDaemon.connectPersistent(ns,Pyro.constants.NAMESERVER_NAME) - else: - ns=NameServer() - PyroDaemon.useNameServer(ns) - NS_URI=PyroDaemon.connect(ns,Pyro.constants.NAMESERVER_NAME) - - BcServerObject = BroadcastServer((hstn or '',bcport),bcRequestHandler) - if Guards[1]: - BcServerObject.setRequestValidator(Guards[1]) - BcServerObject.keepRunning(keep) - if keep: - Log.msg("NS", 'Will ignore shutdown requests.') - ns.ignoreShutdown=True - else: - Log.msg("NS", 'Will accept shutdown requests.') - ns.ignoreShutdown=False - - if Guards[0] or Guards[1]: - print 'Using security plugins:' - if Guards[0]: - print ' NS new conn validator =',Guards[0].__class__.__name__,'from', Guards[0].__class__.__module__ - else: print ' default NS new conn validator' - if Guards[1]: - print ' BC request validator =',Guards[1].__class__.__name__,'from', Guards[1].__class__.__module__ - else: print ' default BC request validator' - - ns.publishURI(NS_URI) - BcServerObject.setNS_URI(NS_URI) - Log.msg('NS daemon','This is the Pyro Name Server.') - if persistent: - Log.msg('NS daemon','Persistent mode, database is in',ns.getDBDir()) - print 'Persistent mode, database is in',ns.getDBDir() - Log.msg('NS daemon','Starting on',PyroDaemon.hostname,'port', PyroDaemon.port, ' broadcast server on port',bcport) - - # I use a timeout here otherwise you can't break gracefully on Windoze - while not BcServerObject.shutdown: - try: - PyroDaemon.handleRequests(BcServerObject.preferredTimeOut,[BcServerObject],BcServerObject.bcCallback) - except KeyboardInterrupt: - Log.warn('NS daemon','shutdown on user break signal') - BcServerObject.shutdown=1 - except: - import traceback - (exc_type, exc_value, exc_trb) = sys.exc_info() - out = ''.join(traceback.format_exception(exc_type, exc_value, exc_trb)[-5:]) - Log.error('NS daemon', 'Unexpected exception, type',exc_type, - '\n--- partial traceback of this exception follows:\n', - out,'\n--- end of traceback') - - Log.msg('NS daemon','Shut down gracefully.') - - -class PyroNSThread(threading.Thread): - """ The Pyro Naming Service will run in this thread - """ - def __init__(self, args, stopcallback): - threading.Thread.__init__(self) - Log.msg("PyroNSsvc", "initializing") - self._args = list(args) - Log.msg("PyroNSsvc", "args are:",self._args) - self._stopcallback = stopcallback - - def run(self): - self.startPyroNS() - self._stopcallback() - - def startPyroNS(self): - try: - """ ripped out of Pyro.naming and slightly changed to - accomodate not using sys.argv, but self._args instead - """ - Args = Pyro.util.ArgParser() - Args.parse(self._args,'hkn:p:b:d:s:') - try: - Args.getOpt('h') - Log.error("PyroNS_svc",""" -Usage: ns [-h] [-n hostname] [-p port] [-b port] - [-d [databasefile]] [-s securitymodule] - where -p = NS server port - -b = NS broadcast port - -n = non-default server hostname - -d = use persistent database, provide optional storage directory - -s = use given python module with security code - -h = print this help -""") - raise SystemExit - except KeyError: - pass - host = Args.getOpt('n','') - port = int(Args.getOpt('p',Pyro.config.PYRO_NS_PORT)) - bcport = int(Args.getOpt('b',Pyro.config.PYRO_NS_BC_PORT)) - try: - dbdir = Args.getOpt('d') - persistent = 1 - except KeyError: - persistent = 0 - dbdir = None - - # we're running as a service, always ignore remote shutdown requests - keep=1 - - try: - secmod = __import__(Args.getOpt('s'),locals(),globals()) - Guards = (secmod.NSGuard(), secmod.BCGuard()) - except ImportError,x: - Log.msg("NS", 'Error loading security module:',x) - raise SystemExit - except KeyError: - secmod = None - Guards = (None,None) - - if Args.ignored: - Log.warn("PyroNS_svc",'Ignored options:',Args.ignored) - if Args.args: - Log.warn("PyroNS_svc",'Ignored arguments:',Args.args) - - Log.msg("PyroNS_svc","Starting the Name Server.") - startServer(host,port,bcport,keep,persistent,dbdir,Guards) - except Exception,x : - Log.error("NS daemon","COULD NOT START!!!",x) - raise SystemExit - - -class PyroNS_NTService(BasicNTService): - _svc_name_ = 'PyroNS' - _svc_display_name_ = "Pyro Naming Service" - _svc_description_ = 'Provides name resolution services for Pyro objects' - def __init__(self, args): - super(PyroNS_NTService, self).__init__(args) - setConfig() - try: - args = getRegistryParameters(self._svc_name_).split() - except Exception,x: - Log.error("PyroNS_svc","PROBLEM GETTING ARGS FROM REGISTRY:",x) - self._nsThread = PyroNSThread(args, self.SvcStop) - - def _doRun(self): - """ Overriden """ - self._nsThread.start() - - def _doStop(self): - """ Overridden """ - global BcServerObject - BcServerObject.shutdown = 1 - self._nsThread.join() - - def SvcStop(self): - """Overriden """ - super(PyroNS_NTService, self).SvcStop() - - -if __name__ == '__main__': - PyroNS_NTService.HandleCommandLine() diff --git a/lib/Pyro/ext/ServiceTest.py b/lib/Pyro/ext/ServiceTest.py deleted file mode 100644 index 65ebd6b281f..00000000000 --- a/lib/Pyro/ext/ServiceTest.py +++ /dev/null @@ -1,89 +0,0 @@ -############################################################################# -# -# A test for the PyroNS_NTService program -# Author: Syver Enstad syver-en@online.no -# -# This is part of "Pyro" - Python Remote Objects -# Which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -import unittest -import win32serviceutil -import win32service -import time -import Pyro.nsc - -ServiceName = 'PyroNS' - -class Test(unittest.TestCase): - def setUp(self): - win32serviceutil.StartService(ServiceName) - - def testStartPending(self): - svcType, svcState, svcControls, err, svcErr, svcCP, svcWH = \ - win32serviceutil.QueryServiceStatus(ServiceName) - assert svcState & win32service.SERVICE_START_PENDING - - def testFullyStarted(self): - self._waitForStarted() - svcType, svcState, svcControls, err, svcErr, svcCP, svcWH = \ - win32serviceutil.QueryServiceStatus(ServiceName) - assert svcType & win32service.SERVICE_WIN32_OWN_PROCESS - assert svcState & win32service.SERVICE_RUNNING - assert svcControls & win32service.SERVICE_ACCEPT_STOP - - def testStop(self): - self._waitForStarted() - svcType, svcState, svcControls, err, svcErr, svcCP, svcWH = \ - win32serviceutil.StopService(ServiceName) - assert svcState & win32service.SERVICE_STOPPED - assert svcType & win32service.SERVICE_WIN32_OWN_PROCESS - - def testNameserverAvailable(self): - self._waitForStarted() - ctrl = Pyro.nsc.PyroNSControl() - ctrl.args(None) - ctrl.ping() - - def testNameserverShutdownFromNsc(self): - self._waitForStarted() - ctrl = Pyro.nsc.PyroNSControl() - ctrl.args(None) - ctrl.shutdown() - for each in range(100): - svcType, svcState, svcControls, err, svcErr, svcCP, svcWH = \ - win32serviceutil.QueryServiceStatus(ServiceName) - if svcState & win32service.SERVICE_STOPPED: - return - time.sleep(0.20) - self.fail() - - def tearDown(self): - for each in range(1000): - svcType, svcState, svcControls, err, svcErr, svcCP, svcWH = \ - win32serviceutil.QueryServiceStatus(ServiceName) - if svcState & win32service.SERVICE_RUNNING: - svcType, svcState, svcControls, err, svcErr, svcCP, svcWH = \ - win32serviceutil.StopService(ServiceName) - time.sleep(0.1) - elif svcState & win32service.SERVICE_STOPPED: - time.sleep(0.10) - break - else: - time.sleep(0.10) - assert svcState & win32service.SERVICE_STOPPED - time.sleep(3) - - def _waitForStarted(self): - for each in range(100): - svcType, svcState, svcControls, err, svcErr, svcCP, svcWH = \ - win32serviceutil.QueryServiceStatus(ServiceName) - if svcState & win32service.SERVICE_RUNNING: - break - else: - time.sleep(0.10) - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/Pyro/ext/__init__.py b/lib/Pyro/ext/__init__.py deleted file mode 100644 index 6f66f15aed2..00000000000 --- a/lib/Pyro/ext/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# just to make this a package. diff --git a/lib/Pyro/ext/daemonizer.py b/lib/Pyro/ext/daemonizer.py deleted file mode 100644 index 11b50614d46..00000000000 --- a/lib/Pyro/ext/daemonizer.py +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python -############################################################################# -# -# Run Pyro servers as daemon processes on Unix/Linux. -# This won't work on other operating systems such as Windows. -# Author: Jeff Bauer (jbauer@rubic.com) -# This software is released under the MIT software license. -# Based on an earlier daemonize module by Jeffery Kunce -# Updated by Luis Camaano to double-fork-detach. -# -# DEPRECATED. Don't use this in new code. -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -import sys, os, time -from signal import SIGINT - -class DaemonizerException: - def __init__(self, msg): - self.msg = msg - def __str__(self): - return self.msg - -class Daemonizer: - """ - Daemonizer is a class wrapper to run a Pyro server program - in the background as daemon process. The only requirement - is for the derived class to implement a main_loop() method. - See Test class below for an example. - - The following command line operations are provided to support - typical /etc/init.d startup/shutdown on Unix systems: - - start | stop | restart - - In addition, a daemonized program can be called with arguments: - - status - check if process is still running - - debug - run the program in non-daemon mode for testing - - Note: Since Daemonizer uses fork(), it will not work on non-Unix - systems. - """ - def __init__(self, pidfile=None): - if not pidfile: - # PID file moved out of /tmp to avoid security vulnerability - # changed by Debian maintainer per Debian bug #631912 - self.pidfile = "/var/run/pyro-%s.pid" % self.__class__.__name__.lower() - else: - self.pidfile = pidfile - - def become_daemon(self, root_dir='/'): - if os.fork() != 0: # launch child and ... - os._exit(0) # kill off parent - os.setsid() - os.chdir(root_dir) - os.umask(0) - if os.fork() != 0: # fork again so we are not a session leader - os._exit(0) - sys.stdin.close() - sys.__stdin__ = sys.stdin - sys.stdout.close() - sys.stdout = sys.__stdout__ = _NullDevice() - sys.stderr.close() - sys.stderr = sys.__stderr__ = _NullDevice() - for fd in range(1024): - try: - os.close(fd) - except OSError: - pass - - def daemon_start(self, start_as_daemon=1): - if start_as_daemon: - self.become_daemon() - if self.is_process_running(): - msg = "Unable to start server. Process is already running." - raise DaemonizerException(msg) - f = open(self.pidfile, 'w') - f.write("%s" % os.getpid()) - f.close() - self.main_loop() - - def daemon_stop(self): - pid = self.get_pid() - try: - os.kill(pid, SIGINT) # SIGTERM is too harsh... - time.sleep(1) - try: - os.unlink(self.pidfile) - except OSError: - pass - except IOError: - pass - - def get_pid(self): - try: - f = open(self.pidfile) - pid = int(f.readline().strip()) - f.close() - except IOError: - pid = None - return pid - - def is_process_running(self): - pid = self.get_pid() - if pid: - try: - os.kill(pid, 0) - return 1 - except OSError: - pass - return 0 - - def main_loop(self): - """NOTE: This method must be implemented in the derived class.""" - msg = "main_loop method not implemented in derived class: %s" % \ - self.__class__.__name__ - raise DaemonizerException(msg) - - def process_command_line(self, argv, verbose=1): - usage = "usage: %s start | stop | restart | status | debug " \ - "[--pidfile=...] " \ - "(run as non-daemon)" % os.path.basename(argv[0]) - if len(argv) < 2: - print usage - raise SystemExit - else: - operation = argv[1] - if len(argv) > 2 and argv[2].startswith('--pidfile=') and \ - len(argv[2]) > len('--pidfile='): - self.pidfile = argv[2][len('--pidfile='):] - pid = self.get_pid() - if operation == 'status': - if self.is_process_running(): - print "Server process %s is running." % pid - else: - print "Server is not running." - elif operation == 'start': - if self.is_process_running(): - print "Server process %s is already running." % pid - raise SystemExit - else: - if verbose: - print "Starting server process." - self.daemon_start() - elif operation == 'stop': - if self.is_process_running(): - self.daemon_stop() - if verbose: - print "Server process %s stopped." % pid - else: - print "Server process %s is not running." % pid - raise SystemExit - elif operation == 'restart': - self.daemon_stop() - if verbose: - print "Restarting server process." - self.daemon_start() - elif operation == 'debug': - self.daemon_start(0) - else: - print "Unknown operation:", operation - raise SystemExit - - -class _NullDevice: - """A substitute for stdout/stderr that writes to nowhere.""" - def write(self, s): - pass - - -class Test(Daemonizer): - def __init__(self): - Daemonizer.__init__(self) - - def main_loop(self): - while 1: - time.sleep(1) - - -if __name__ == "__main__": - test = Test() - test.process_command_line(sys.argv) diff --git a/lib/Pyro/ext/remote.py b/lib/Pyro/ext/remote.py deleted file mode 100644 index e446109bac3..00000000000 --- a/lib/Pyro/ext/remote.py +++ /dev/null @@ -1,499 +0,0 @@ -############################################################################# -# -# simple Pyro connection module, originally written by John Wiegley -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -import UserDict -import exceptions -import os -import re -import signal -import socket -import sys -import time -import types - -import Pyro.errors -import Pyro.naming -import Pyro.core -import Pyro.util -from Pyro.util import HostUtil - -from Pyro.protocol import ProtocolError - -true, false = 1, 0 - -copy_types = false -verbose = false -pyro_nameserver = None -pyro_daemon = None -client_initialized = false -server_initialized = false -daemon_host = '' -daemon_port = 0 -daemon_objects = [] -daemon_types = [] - -def tb_info(tb): - codename = tb.tb_frame.f_code.co_filename - lineno = tb.tb_lineno - if not (codename == '' or codename.find(".py") > 0): - lineno = lineno - 2 - return lineno, codename - -def canonize(e_type, e_val, e_traceback): - "Turn the exception into a textual representation." - # find the last traceback: - tb = e_traceback - - lineno, codename = tb_info(tb) - lines = [ "%s %s" % (codename, lineno) ] - - found = None - if tb.tb_frame.f_code.co_filename[0] == '<': - found = tb - - while tb.tb_next: - tb = tb.tb_next - if tb.tb_frame.f_code.co_filename[0] == '<': - found = tb - lineno, codename = tb_info(tb) - lines.append("%s %s" % (codename, lineno)) - - if found: - tb = found - - lineno, codename = tb_info(tb) - - if codename == '': - lines.insert(0, "%s in command: %s" % (e_type, e_val)) - elif codename.find(".py") > 0 and e_type == "SyntaxError": - lines.insert(0, "%s in: %s" % (e_type, e_val)) - else: - lines.insert(0, "%s in line %s of %s: %s" % - (e_type, lineno, codename, e_val)) - - return lines - -def exception_text(): - return sys.exc_value - -def format_exception(): - return canonize(*sys.exc_info()) - -def register_type(t): - """Whenever type T goes in or out, wrap/unwrap the type so that - the user is always interacting with the server object, or the - server interacts with the object directly.""" - if t not in daemon_types: - daemon_types.append(t) - -def unregister_objects(): - if pyro_daemon: - global daemon_objects - for obj in daemon_objects: - try: pyro_daemon.disconnect(obj) - except: pass - daemon_objects = [] - -sys.exitfunc = unregister_objects - -def host_ipaddr(interface = None): - if sys.platform == "win32": - return HostUtil.get_inst().gethostbyname() - - cmd = "/sbin/ifconfig" - if interface: - cmd = '%s %s' % (cmd, interface) - fd = os.popen(cmd) - - this_host = None - interfaces = {} - name = None - - for line in fd.readlines(): - match = re.match("(\S+)", line) - if match: name = match.group(1) - match = re.search("inet addr:(\S+)", line) - if match: - addr = match.group(1) - if name: - interfaces[name] = addr - - if interfaces.has_key(interface): - this_host = interfaces[interface] - else: - for name, addr in interfaces.items(): - if re.match("ppp", name): - this_host = addr - break - elif re.match("eth", name): - this_host = addr - - fd.close() - - return this_host or HostUtil.get_inst().gethostbyname() - -def find_nameserver(hostname = None, portnum = None): - if hostname and hostname.find('://') > 0: - URI = Pyro.core.PyroURI(hostname) - ns = Pyro.naming.NameServerProxy(URI) - else: - try: - if verbose: - print 'Searching for Naming Service on %s:%d...' % \ - (hostname or 'BROADCAST', - portnum or Pyro.config.PYRO_NS_BC_PORT) - locator = Pyro.naming.NameServerLocator() - ns = locator.getNS(host = hostname, port = portnum) - - except (Pyro.core.PyroError, socket.error), x: - localhost = HostUtil.get_inst().gethostbyname('localhost') - if verbose: - print "Error:", x - print """ -Naming Service not found with broadcast. -Trying local host""", localhost, '...', - - ns = locator.getNS(host = localhost, port = portnum) - - if verbose: print 'Naming Service found at', ns.URI - - return ns - -class Error(Exception): pass - -class ObjBase(Pyro.core.ObjBase): - """This extension of Pyro.core.ObjBase makes sure that any values - that get returned to the caller which are of a significant type, - get wrapped first in proxies. - - Likewise, if a proxy class comes back to us, and it's in regard to - an object native to this server, unwrap it.""" - def __nonzero__(self): return 1 - - def Pyro_dyncall(self, method, flags, args): - try: - base = Pyro.core.ObjBase.Pyro_dyncall - result = wrap(base(self, method, flags, unwrap(args))) - except: - result = Error('\n'.join(format_exception())) - return result - - def _r_ga(self, attr): - return wrap(Pyro.core.ObjBase._r_ga(self, attr)) - - def _r_sa(self, attr, value): - Pyro.core.ObjBase._r_sa(self, attr, unwrap(value)) - -class Nameserver: - """This helper class allows the server to use Pyro's naming - service for publishing certain objects by name. It integrates - better with remote.py, than Pyro.naming.NameServer does.""" - def __init__(self, ns, ns_port): - self.ns = ns - self.ns_port = ns_port - - def __cmp__(self, other): - return self.ns == other.ns and self.ns_port == other.ns_port - - def __str__(self): - if self.ns_port: - return "%s:%s" % (self.ns, self.ns_port) - return self.ns - - def resolve(self, name): - return get_remote_object(name, self.ns, self.ns_port) - - def register(self, name, object): - return provide_local_object(object, name, self.ns, self.ns_port) - - def unregister(self, object): - for obj in daemon_objects[:]: - if obj.delegate is object: - pyro_daemon.disconnect(obj) - daemon_objects.remove(obj) - -class DynamicProxy(Pyro.core.DynamicProxyWithAttrs): - """This version of the proxy just wraps args before making - external calls.""" - def __nonzero__(self): - return true - - def _invokePYRO(self, *vargs, **kargs): - result = unwrap(apply(Pyro.core.DynamicProxyWithAttrs._invokePYRO, - tuple([self] + wrap(list(vargs))), wrap(kargs))) - - if type(result) is types.InstanceType and \ - isinstance(result, Error) or \ - isinstance(result, Pyro.errors.PyroError) or \ - isinstance(result, ProtocolError): - msg = str(result) - type_name = msg[: msg.find(' ')] - - if type_name == 'exceptions.IndexError': - try: - real_type = eval(type_name) - msg = msg.split('\n')[0] - result = real_type(msg[msg.find(':') + 2 :]) - except: - pass - - raise result - else: - return result - -def unwrap(value): - t = type(value) - if t is types.InstanceType and isinstance(value, DynamicProxy): - if pyro_daemon: - try: - return pyro_daemon.getLocalObject(value.objectID) - except KeyError: - pass - return value - elif t is types.ListType: - for i in range(len(value)): - value[i] = unwrap(value[i]) - elif t is types.TupleType: - value = list(value) - for i in range(len(value)): - value[i] = unwrap(value[i]) - return tuple(value) - elif t is types.DictType: - for k, v in value.items(): - value[k] = unwrap(v) - return value - -def wrap(value): - """Wrap the argument, returning a copy -- since otherwise we might - alter a local data structure inadvertantly.""" - t = type(value) - if t is types.InstanceType: - matched = false - for dt in daemon_types: - if isinstance(value, dt): - matched = true - if not copy_types and not matched and \ - not isinstance(value, DynamicProxy): - return provide_local_object(value) - elif t is types.ListType: - value = value[:] - for i in range(len(value)): - value[i] = wrap(value[i]) - elif t is types.TupleType: - value = list(value) - for i in range(len(value)): - value[i] = wrap(value[i]) - return tuple(value) - elif t is types.DictType: - copy = {} - for k, v in value.items(): - copy[k] = wrap(v) - return copy - return value - -def get_remote_object(name, hostname = None, portnum = None): - global client_initialized, pyro_nameserver - - # initialize Pyro -- Python Remote Objects - if not client_initialized: - Pyro.core.initClient(verbose) - client_initialized = true - - if pyro_nameserver is None or hostname: - pyro_nameserver = find_nameserver(hostname, portnum) - - if verbose: - print 'Binding object %s' % name - - try: - URI = pyro_nameserver.resolve(name) - if verbose: - print 'URI:', URI - - return DynamicProxy(URI) - - except Pyro.core.PyroError, x: - raise Error("Couldn't bind object, nameserver says:", x) - -class Cache(UserDict.UserDict): - """simple cache that uses least recently accessed time to trim size""" - def __init__(self,data=None,size=100): - UserDict.UserDict.__init__(self,data) - self.size = size - - def resize(self): - """trim cache to no more than 95% of desired size""" - trim = max(0, int(len(self.data)-0.95*self.size)) - if trim: - # don't want self.items() because we must sort list by access time - values = map(None, self.data.values(), self.data.keys()) - values.sort() - for val,k in values[0:trim]: - del self.data[k] - - def __setitem__(self,key,val): - if (not self.data.has_key(key) and - len(self.data) >= self.size): - self.resize() - self.data[key] = (time.time(), val) - - def __getitem__(self,key): - """like normal __getitem__ but updates time of fetched entry""" - val = self.data[key][1] - self.data[key] = (time.time(),val) - return val - - def get(self,key,default=None): - """like normal __getitem__ but updates time of fetched entry""" - try: - return self[key] - except KeyError: - return default - - def values(self): - """return values, but eliminate access times first""" - vals = list(self.data.values()) - for i in range(len(vals)): - vals[i] = vals[i][1] - return tuple(vals) - - def items(self): - return map(None, self.keys(), self.values()) - - def copy(self): - return self.__class__(self.data, self.size) - - def update(self, otherdict): - for k in otherdict.keys(): - self[k] = otherdict[k] - -provided_objects = Cache(size = 100) - -def provide_local_object(obj, name = None, hostname = None, portnum = None): - global server_initialized, pyro_daemon, pyro_nameserver - - proxy_class = DynamicProxy - - if not server_initialized: - Pyro.core.initServer(verbose) - server_initialized = true - - if pyro_daemon is None: - pyro_daemon = Pyro.core.Daemon(host = daemon_host, - port = daemon_port) - - # If no 'name' was specified, don't even bother with the - # nameserver. - if name: - if pyro_nameserver is None or hostname: - pyro_nameserver = find_nameserver(hostname, portnum) - - pyro_daemon.useNameServer(pyro_nameserver) - - if verbose: - print 'Remoting object', name - - # tell nameserver to forget any earlier use of this name - try: - if pyro_nameserver.resolve(name): - pyro_nameserver.unregister(name) - except Pyro.errors.NamingError: - pass - - if not isinstance(obj, Pyro.core.ObjBase): - if provided_objects.has_key(obj): - obj = provided_objects[obj] - else: - slave = ObjBase() - slave.delegateTo(obj) - provided_objects[obj] = slave - obj = slave - - URI = pyro_daemon.connect(obj, name) - daemon_objects.append(obj) - - proxy = proxy_class(URI) - - return proxy - -abort = false - -def interrupt(*args): - global abort - abort = true - -if hasattr(signal,'SIGINT'): signal.signal(signal.SIGINT, interrupt) -#if hasattr(signal,'SIGHUP'): signal.signal(signal.SIGHUP, interrupt) -#if hasattr(signal,'SIGQUIT'): signal.signal(signal.SIGQUIT, interrupt) - -def handle_requests(wait_time = None, callback = None): - global abort - - abort = false - - if pyro_daemon is None: - raise Error("There is no daemon with which to handle requests") - - if wait_time: - start = time.time() - - while not abort: - try: - pyro_daemon.handleRequests(wait_time) - - if wait_time: - now = time.time() - if callback and now - start > wait_time: - callback() - start = now - elif callback: - callback() - - # ignore socket and select errors, they are often transient - except socket.error: pass - except Exception, msg: - if verbose: - print "Error:", sys.exc_type, msg - abort = true - except: - abort = true - - return abort - -def handle_requests_unsafe(wait_time = None, callback = None): - global abort - - abort = false - - if pyro_daemon is None: - raise Error("There is no daemon with which to handle requests") - - if wait_time: - start = time.time() - - while 1: - pyro_daemon.handleRequests(wait_time) - - if wait_time: - now = time.time() - if callback and now - start > wait_time: - callback() - start = now - elif callback: - callback() - - return true - -def unregister_object(obj): - if pyro_daemon: - try: pyro_daemon.disconnect(obj) - except: pass - global daemon_objects - if obj in daemon_objects: - daemon_objects.remove(obj) diff --git a/lib/Pyro/ext/remote_nons.py b/lib/Pyro/ext/remote_nons.py deleted file mode 100644 index a40bdea0a36..00000000000 --- a/lib/Pyro/ext/remote_nons.py +++ /dev/null @@ -1,119 +0,0 @@ -############################################################################# -# -# simple Pyro connection module, without requiring Pyro's NameServer -# (adapted from John Wiegley's remote.py) -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -import signal -import sys -import time - -import Pyro.errors -import Pyro.naming -import Pyro.core -import Pyro.util - -true, false = 1, 0 - -verbose = false -pyro_daemon = None -client_initialized = false -server_initialized = false -daemon_objects = [] - -from Pyro.protocol import ProtocolError - - -def get_server_object(objectName, hostname , portnum): - global client_initialized - - # initialize Pyro -- Python Remote Objects - if not client_initialized: - Pyro.core.initClient(verbose) - client_initialized = true - - - if verbose: - print 'Binding object %s' % objectName - - try: - URI = 'PYROLOC://%s:%d/%s' % (hostname,portnum,objectName) - if verbose: - print 'URI:', URI - - return Pyro.core.getAttrProxyForURI(URI) - - except Pyro.core.PyroError, x: - raise Pyro.core.PyroError("Couldn't bind object, Pyro says:", x) - -def provide_server_object(obj, name = None, hostname = '', portnum = None): - global server_initialized, pyro_daemon - proxy_class = Pyro.core.DynamicProxyWithAttrs - - if not server_initialized: - Pyro.core.initServer(verbose) - server_initialized = true - - if pyro_daemon is None: - pyro_daemon = Pyro.core.Daemon(host = hostname, port = portnum) - - - if not isinstance(obj, Pyro.core.ObjBase): - slave = Pyro.core.ObjBase() - slave.delegateTo(obj) - obj = slave - - URI = pyro_daemon.connect(obj, name) - if verbose: - print 'provide_server_object: URI = ', URI - daemon_objects.append(obj) - - proxy = proxy_class(URI) - - return proxy - -abort = false - -def interrupt(*args): - global abort - abort = true - -if hasattr(signal,'SIGINT'): signal.signal(signal.SIGINT, interrupt) -#if hasattr(signal,'SIGHUP'): signal.signal(signal.SIGHUP, interrupt) -#if hasattr(signal,'SIGQUIT'): signal.signal(signal.SIGQUIT, interrupt) - -def handle_requests(wait_time = None, callback = None): - global abort - - abort = false - - if pyro_daemon is None: - raise Pyro.errors.PyroError("There is no daemon with which to handle requests") - return - - if wait_time: - start = time.time() - - while not abort: - try: - pyro_daemon.handleRequests(wait_time) - if wait_time: - now = time.time() - if callback and now - start > wait_time: - callback() - start = now - elif callback: - callback() - - except Exception, msg: - if verbose: - print "Error:", sys.exc_type, msg - abort = true - except: - abort = true - - return abort diff --git a/lib/Pyro/naming.py b/lib/Pyro/naming.py deleted file mode 100644 index 876d6430e40..00000000000 --- a/lib/Pyro/naming.py +++ /dev/null @@ -1,1490 +0,0 @@ -############################################################################# -# -# Pyro Name Server -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -from __future__ import with_statement -import sys, os, socket, time, traceback, errno -import dircache, shutil, SocketServer -import Pyro.constants, Pyro.core, Pyro.errors, Pyro.protocol, Pyro.util -from Pyro.util import HostUtil -if Pyro.util.supports_multithreading(): - import threading - -NS_SYSCMD_LOCATION='location' -NS_SYSCMD_SHUTDOWN='shutdown' - -Log = Pyro.util.Log - -############################################################################# -# -# The Pyro NameServer Locator. -# Use a broadcast mechanism to find the broadcast server of the NS which -# can provide us with the URI of the NS. -# Can also perform direct lookup (no broadcast) if the host is specified. -# (in that case, the 'port' argument is the Pyro port, not a broadcast port). -# -############################################################################# - -class NameServerLocator(object): - def __init__(self, identification=None): - Pyro.core._checkInit() # init required - self.identification=identification - - def sendSysCommand(self,request,host=None,port=None,trace=0,logerrors=1,bcaddr=None): - try: - # Try the 'first' name server. - # Note that if no host is specified, a broadcast is used, - # and that one is sent to both name servers in parallel. - return self.__sendSysCommand(request, host, port, trace, logerrors, Pyro.constants.NSROLE_PRIMARY, bcaddr) - except KeyboardInterrupt: - raise - except (socket.error, Pyro.errors.PyroError): - if not port: - # the 'first' name server failed, try the second - try: - result=self.__sendSysCommand(request, host, port, trace, logerrors, Pyro.constants.NSROLE_SECONDARY, bcaddr) - # found the second! - # switch config for first and second so that the second one (which we found) will now be tried first - Pyro.config.PYRO_NS2_HOSTNAME, Pyro.config.PYRO_NS_HOSTNAME = Pyro.config.PYRO_NS_HOSTNAME, Pyro.config.PYRO_NS2_HOSTNAME - Pyro.config.PYRO_NS2_PORT, Pyro.config.PYRO_NS_PORT = Pyro.config.PYRO_NS_PORT, Pyro.config.PYRO_NS2_PORT - Pyro.config.PYRO_NS2_BC_PORT, Pyro.config.PYRO_NS_BC_PORT = Pyro.config.PYRO_NS_BC_PORT, Pyro.config.PYRO_NS2_BC_PORT - Pyro.config.PYRO_NS2_BC_ADDR, Pyro.config.PYRO_NS_BC_ADDR = Pyro.config.PYRO_NS_BC_ADDR, Pyro.config.PYRO_NS2_BC_ADDR - return result - except (socket.error, Pyro.errors.PyroError): - # Could not find using broadcast. Try the current host and localhost as well. - # But only if there's no explicit host parameter given. - if host: - raise Pyro.errors.NamingError("could not find NameServer on host "+host) - else: - for host in (Pyro.protocol.getHostname(), "localhost"): - if trace: - print "Trying host",host - Log.msg('NameServerLocator','Trying host',host) - try: - result=self.__sendSysCommand(request, host, port, trace, logerrors, Pyro.constants.NSROLE_PRIMARY) - Pyro.config.PYRO_NS_HOSTNAME = host - return result - except Pyro.errors.ConnectionDeniedError: - raise - except (socket.error, Pyro.errors.PyroError),x: - pass - else: - raise Pyro.errors.NamingError("could not find NameServer") - else: - raise - - def __sendSysCommand(self,request,host=None,port=None,trace=0,logerrors=1,role=Pyro.constants.NSROLE_PRIMARY,bcaddr=None): - HPB={Pyro.constants.NSROLE_PRIMARY: (Pyro.config.PYRO_NS_HOSTNAME, Pyro.config.PYRO_NS_PORT, Pyro.config.PYRO_NS_BC_PORT, Pyro.config.PYRO_NS_BC_ADDR), - Pyro.constants.NSROLE_SECONDARY: (Pyro.config.PYRO_NS2_HOSTNAME, Pyro.config.PYRO_NS2_PORT, Pyro.config.PYRO_NS2_BC_PORT, Pyro.config.PYRO_NS2_BC_ADDR) } - if not host: - host=HPB[role][0] - if port: - port1=port2=port - else: - if not host: - # select the default broadcast ports - port1 = HPB[Pyro.constants.NSROLE_PRIMARY][2] - port2 = HPB[Pyro.constants.NSROLE_SECONDARY][2] - else: - # select the default port (normal) - port = HPB[role][1] - # We must discover the location of the name server. - # Pyro's NS can answer to broadcast requests. - try: - if host: - # use direct lookup with PYROLOC: mechanism, no broadcast - if trace: - print 'Locator: contacting Pyro Name Server...' - uri=Pyro.core.PyroURI(host,Pyro.constants.NAMESERVER_NAME,port,'PYROLOC') - prox=Pyro.core.getProxyForURI(uri) - prox._setIdentification(self.identification) - if request==NS_SYSCMD_LOCATION: - prox.ping() # force resolving of PYROLOC: uri - return prox.URI # return resolved uri - elif request==NS_SYSCMD_SHUTDOWN: - return prox._shutdown() - else: - raise ValueError("invalid command specified") - - # No host specified. Use broadcast mechanism - if os.name=='java' and sys.version_info<(2,5): - # jythons older than 2.5 don't have working broadcast - msg="Skipping UDP broadcast (older jythons don't support this operation)" - if trace: - print msg - raise Pyro.errors.PyroError(msg) - if bcaddr: - try: - HostUtil.get_inst().gethostbyname(bcaddr) - except socket.error: - msg="invalid broadcast address '%s'" % bcaddr - if trace: - print msg - raise ValueError(msg) - destination1 = (bcaddr, port1) - destination2 = (bcaddr, port2) - else: - destination1 = (Pyro.config.PYRO_NS_BC_ADDR or '', port1) - destination2 = (Pyro.config.PYRO_NS2_BC_ADDR or '', port2) - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - if hasattr(socket,'SO_BROADCAST'): - s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - - if trace: - print 'Locator: searching Pyro Name Server...' - try: - bc_retries=Pyro.config.PYRO_BC_RETRIES - if bc_retries<0: - bc_retries=sys.maxint-1 - bc_retries = min(sys.maxint-1, bc_retries) - for i in xrange(bc_retries+1): - # send request to both Pyro NS (if running in paired mode) - s.sendto(request, destination1) - if destination2!=destination1: - s.sendto(request, destination2) - timeout=min(sys.maxint,Pyro.config.PYRO_BC_TIMEOUT) - if timeout<0: - timeout=None - ins,outs,exs = Pyro.protocol.safe_select([s],[],[s],timeout) - if s in ins: - # return the info of the first NS that responded. - reply, fromaddr = s.recvfrom(1000) - return reply - if trace and i') - self.lock=Pyro.util.getLockObject() - self.role=role - self.otherNS=None - self.ignoreShutdown=False - if role in (Pyro.constants.NSROLE_PRIMARY, Pyro.constants.NSROLE_SECONDARY): - # for paired mode with identification, we need to remember the ident string - adapter=Pyro.protocol.getProtocolAdapter("PYRO") - adapter.setIdentification(identification) - self.identification=adapter.getIdentification() # grab the munged ident - # create default groups - self.createGroup(':'+'Pyro') - self.createGroup(Pyro.config.PYRO_NS_DEFAULTGROUP) - Log.msg("NameServer","Running in", - {Pyro.constants.NSROLE_SINGLE:"single", - Pyro.constants.NSROLE_PRIMARY:"primary", - Pyro.constants.NSROLE_SECONDARY:"secondary"}[self.role],"mode" ) - - def _initialResyncWithTwin(self, twinProxy): - if twinProxy: - try: - Log.msg("NameServer","Initial resync with other NS at",twinProxy.URI.address,"port",twinProxy.URI.port) - print "Initial Resync with other NS at",twinProxy.URI.address,"port",twinProxy.URI.port - # keep old NS (self) registration - oldNSreg=self.resolve(Pyro.constants.NAMESERVER_NAME) - proxyForMe=NameServerProxy(self.getProxy().URI,noconnect=1) - proxyForMe.adapter.setIdentification(self.identification,munge=False) # set pre-munged ident - self.root=twinProxy._resync(proxyForMe) - # reset self registration - try: - self.unregister(Pyro.constants.NAMESERVER_NAME) - except: - pass - self.register(Pyro.constants.NAMESERVER_NAME,oldNSreg) - self.otherNS=twinProxy - Log.msg("NameServer","database sync complete.") - print "Database synchronized." - except Pyro.errors.NamingError,x: - print x - raise - - def _removeTwinNS(self): - self.otherNS=None - - def register(self,name,URI): - (origname,name)=name,self.validateName(name) - URI=self.validateURI(URI) - with self.lock: - (group, name)=self.locateGrpAndName(name) - if len(name or "")<1: - raise Pyro.errors.NamingError('invalid name',origname) - if isinstance(group,NameValue): - raise Pyro.errors.NamingError('parent is no group', group.name) - try: - group.newleaf(name,URI) - Log.msg('NameServer','registered',name,'with URI',str(URI)) - self._dosynccall("register",origname,URI) - except KeyError: - Log.msg('NameServer','name already exists:',name) - raise Pyro.errors.NamingError('name already exists',name) - - def unregister(self,name): - (origname,name)=name,self.validateName(name) - with self.lock: - (group, name)=self.locateGrpAndName(name) - if len(name or "")<1: - raise Pyro.errors.NamingError('invalid name',origname) - try: - group.cutleaf(name) - Log.msg('NameServer','unregistered',name) - self._dosynccall("unregister",origname) - except KeyError: - raise Pyro.errors.NamingError('name not found',name) - except ValueError: - Log.msg('NameServer','attempt to remove a group:',name) - raise Pyro.errors.NamingError('is a group, not an object',name) - - def resolve(self,name): - # not thread-locked: higher performance and not necessary. - name=self.validateName(name) - try: - branch=self.getBranch(name) - if isinstance(branch,NameValue): - return branch.value - else: - Log.msg('NameServer','attempt to resolve groupname:',name) - raise Pyro.errors.NamingError('attempt to resolve groupname',name) - except KeyError: - raise Pyro.errors.NamingError('name not found',name) - except AttributeError: - raise Pyro.errors.NamingError('group not found',name) - - def flatlist(self): - # return a dump - with self.lock: - r=self.root.flatten() - for i in xrange(len(r)): - r[i]=(':'+r[i][0], r[i][1]) - return r - - def ping(self): - # Just accept a remote invocation. - # This method is used to check if NS is still running, - # and also by the locator if a direct lookup is needed. - pass - - # --- sync support (twin NS) - def _resync(self, twinProxy): - if self.role!=Pyro.constants.NSROLE_SINGLE: - Log.msg("NameServer","resync requested from NS at",twinProxy.URI.address,"port",twinProxy.URI.port) - print "Resync requested from NS at",twinProxy.URI.address,"port",twinProxy.URI.port - self.otherNS=twinProxy - with self.lock: - return self._getSyncDump() - else: - Log.warn("NameServer","resync requested from",twinProxy.URI,"but not running in correct mode") - raise Pyro.errors.NamingError("The (other) NS is not running in 'primary' or 'secondary' mode") - - # remotely called: - def _synccall(self, method, *args): - # temporarily disable the other NS - oldOtherNS, self.otherNS = self.otherNS, None - getattr(self, method) (*args) - self.otherNS = oldOtherNS - - def resync(self): - if self.role==Pyro.constants.NSROLE_SINGLE: - raise Pyro.errors.NamingError("NS is not running in 'primary' or 'secondary' mode") - if self.otherNS: - try: - self._initialResyncWithTwin(self.otherNS) - return - except Exception: - pass - raise Pyro.errors.NamingError("cannot resync: twin NS is unknown or unreachable") - - # local helper: - def _dosynccall(self, method, *args): - if self.role!=Pyro.constants.NSROLE_SINGLE and self.otherNS: - try: - self.otherNS._synccall(method, *args) - except Exception,x: - Log.warn("NameServer","ignored error in _synccall - but removing other NS",x) - self.otherNS=None - - # --- hierarchical naming support - def createGroup(self,groupname): - groupname=self.validateName(groupname) - if len(groupname)<2: - raise Pyro.errors.NamingError('invalid groupname', groupname) - with self.lock: - (parent,name)=self.locateGrpAndName(groupname) - if isinstance(parent,NameValue): - raise Pyro.errors.NamingError('parent is no group', groupname) - try: - parent.newbranch(name) - Log.msg('NameServer','created group',groupname) - self._dosynccall("createGroup",groupname) - except KeyError: - raise Pyro.errors.NamingError('group already exists',name) - - def deleteGroup(self,groupname): - groupname=self.validateName(groupname) - if groupname==':': - Log.msg('NameServer','attempt to deleteGroup root group') - raise Pyro.errors.NamingError('not allowed to delete root group') - with self.lock: - (parent,name)=self.locateGrpAndName(groupname) - try: - parent.cutbranch(name) - Log.msg('NameServer','deleted group',name) - self._dosynccall("deleteGroup",groupname) - except KeyError: - raise Pyro.errors.NamingError('group not found',groupname) - except ValueError: - raise Pyro.errors.NamingError('is no group',groupname) - - def list(self,groupname): - # not thread-locked: higher performance and not necessary. - if not groupname: - groupname=':' - groupname=self.validateName(groupname) - try: - return self.getBranch(groupname).list() - except KeyError: - raise Pyro.errors.NamingError('group not found',groupname) - except AttributeError: - raise Pyro.errors.NamingError('is no group',groupname) - - # --- meta info support - def setMeta(self, name, meta): - name=self.validateName(name) - try: - branch=self.getBranch(name) - branch.setMeta(meta) - self._dosynccall("setMeta",name,meta) - except KeyError: - raise Pyro.errors.NamingError('name not found',name) - except AttributeError: - raise Pyro.errors.NamingError('group not found',name) - - def getMeta(self, name): - name=self.validateName(name) - try: - branch=self.getBranch(name) - return branch.getMeta() - except KeyError: - raise Pyro.errors.NamingError('name not found',name) - except AttributeError: - raise Pyro.errors.NamingError('group not found',name) - - def _setSystemMeta(self, name, meta): - name=self.validateName(name) - try: - branch=self.getBranch(name) - branch.setSystemMeta(meta) - self._dosynccall("_setSystemMeta",name,meta) - except KeyError: - raise Pyro.errors.NamingError('name not found',name) - except AttributeError: - raise Pyro.errors.NamingError('group not found',name) - - def _getSystemMeta(self, name): - name=self.validateName(name) - try: - branch=self.getBranch(name) - return branch.getSystemMeta() - except KeyError: - raise Pyro.errors.NamingError('name not found',name) - except AttributeError: - raise Pyro.errors.NamingError('group not found',name) - - # --- shut down the server - def _shutdown(self): - if self.ignoreShutdown: - Log.msg('NameServer','received shutdown request, but shutdown is denied') - return 'Shutdown request denied' - else: - Log.msg('NameServer','received shutdown request, will shutdown shortly') - self.getDaemon().shutdown() - return "Will shut down shortly" - - # --- private methods follow - def _getSyncDump(self): - return self.root - - def locateGrpAndName(self,name): - # ASSUME name is absolute (from root) (which is required here) - idx=name.rfind('.') - if idx>=0: - # name is hierarchical - grpname=name[:idx] - name=name[idx+1:] - try: - return (self.getBranch(grpname), name) - except KeyError: - raise Pyro.errors.NamingError('(parent)group not found',grpname) - else: - # name is in root - return (self.root, name[1:]) - - def getBranch(self,name): - # ASSUME name is absolute (from root) (which is required here) - name=name[1:] - if name: - return reduce(lambda x,y: x[y], name.split('.'), self.root) - else: - return self.root - - def validateName(self,name): - if name[0]==':': - if ('' not in name.split('.')): - for i in name: - if ord(i)<33 or ord(i)>126 or i=='\\': - raise Pyro.errors.NamingError('invalid character(s) in name',name) - return name - else: - raise Pyro.errors.NamingError('invalid name',name) - else: - # name is not absolute. Make it absolute. - return _expandName(name) - - def validateURI(self,URI): - if isinstance(URI, Pyro.core.PyroURI): - return URI - try: - return Pyro.core.PyroURI(URI) - except: - raise Pyro.errors.NamingError('invalid URI',URI) - - def publishURI(self, uri, verbose=0): - # verbose is not used - always prints the uri. - uri=str(uri) - print 'URI is:',uri - try: - f=open(Pyro.config.PYRO_NS_URIFILE,'w') - f.write(uri+'\n'); f.close() - print 'URI written to:',Pyro.config.PYRO_NS_URIFILE - Log.msg('NameServer','URI written to',Pyro.config.PYRO_NS_URIFILE) - except: - Log.warn('NameServer','Couldn\'t write URI to',Pyro.config.PYRO_NS_URIFILE) - -############################################################################# -# -# NamedTree data type. Used for the hierarchical name server. -# -############################################################################# - -class NameSpaceSystemMeta(object): - def __init__(self, node, timestamp, owner): - self.timestamp=timestamp - self.owner=owner - if isinstance(node, NamedTree): - self.type=0 # tree - else: - self.type=1 # leaf - def __str__(self): - return "[type="+str(self.type)+" timestamp="+str(self.timestamp)+" owner="+str(self.owner)+"]" - - -# All nodes in the namespace (groups, or namevalue pairs--leafs) have -# a shared set of properties, most notably: meta information. -class NameSpaceNode(object): - def __init__(self, name, meta, owner): - self.name=name - self.systemMeta = NameSpaceSystemMeta(self, time.time(), owner) - self.userMeta = meta - def getMeta(self): - return self.userMeta - def getSystemMeta(self): - return self.systemMeta - def setMeta(self,meta): - self.userMeta=meta - def setSystemMeta(self,meta): - if isinstance(meta, NameSpaceSystemMeta): - self.systemMeta=meta - else: - raise TypeError("system meta info must be NameSpaceSystemMeta object") - -class NameValue(NameSpaceNode): - def __init__(self, name, value=None, meta=None, owner=None): - NameSpaceNode.__init__(self, name, meta, owner) - self.value=value - -class NamedTree(NameSpaceNode): - def __init__(self, name, meta=None, owner=None): - NameSpaceNode.__init__(self, name, meta, owner) - self.branches={} - def newbranch(self,name): - if name in self.branches.keys(): - raise KeyError,'name already exists' - t = NamedTree(name) - self.branches[name]=t - return t - def newleaf(self,name,value=None): - if name in self.branches.keys(): - raise KeyError,'name already exists' - l = NameValue(name,value) - self.branches[name]=l - return l - def cutleaf(self,name): - if isinstance(self.branches[name], NameValue): - del self.branches[name] - else: - raise ValueError,'not a leaf' - def cutbranch(self,name): - if isinstance(self.branches[name], NamedTree): - del self.branches[name] - else: - raise ValueError,'not a branch' - def __getitem__(self,name): - return self.branches[name] - def list(self): - l=[] - for (k,v) in self.branches.items(): - if isinstance(v, NamedTree): - l.append( (k,0) ) # tree - elif isinstance(v, NameValue): - l.append( (k,1) ) # leaf - else: - raise ValueError('corrupt tree') - return l - def flatten(self,prefix=''): - flat=[] - for (k,v) in self.branches.items(): - if isinstance(v, NameValue): - flat.append( (prefix+k, v.value) ) - elif isinstance(v, NamedTree): - flat.extend(v.flatten(prefix+k+'.')) - return flat - - - -############################################################################# -# -# The Persistent Name Server (a Pyro Object). -# This implementation uses the hierarchical file system to -# store the groups (as directories) and objects (as files). -# -############################################################################# - -_PNS_META_SUFFIX=".ns_meta" - -class PersistentNameServer(NameServer): - def __init__(self, dbdir=None, role=Pyro.constants.NSROLE_SINGLE, identification=None): - self.dbroot=os.path.join(Pyro.config.PYRO_STORAGE,dbdir or 'Pyro_NS_database') - self._initdb_1() - try: - NameServer.__init__(self, role=role, identification=identification) - except Pyro.errors.NamingError: - pass - self._initdb_2() - - def _initdb_1(self): - # root is not a NamedTree but a directory - try: - os.mkdir(self.dbroot) - except OSError,x: - if x.errno not in (errno.EEXIST, errno.EBUSY): - raise - def _initdb_2(self): - # make sure that the 2 initial groups (Pyro and Default) exist - try: self.createGroup(':'+'Pyro') - except Pyro.errors.NamingError: pass - try: self.createGroup(Pyro.config.PYRO_NS_DEFAULTGROUP) - except Pyro.errors.NamingError: pass - - def getDBDir(self): - return self.dbroot - - def _initialResyncWithTwin(self, twinProxy): - if twinProxy: - Log.msg("NameServer","Initial resync with other NS at",twinProxy.URI.address,"port",twinProxy.URI.port) - # keep old NS (self) registration - oldNSreg=self.resolve(Pyro.constants.NAMESERVER_NAME) - proxyForMe=NameServerProxy(self.getProxy().URI,noconnect=1) - proxyForMe.adapter.setIdentification(self.identification,munge=False) # set pre-munged ident - syncdump=twinProxy._resync(proxyForMe) - self.otherNS = None # temporarily disable twin NS ref - # clear the old database - Log.msg("NameServer","erasing old database",self.dbroot) - shutil.rmtree(self.dbroot) - self._initdb_1() # phase 2 (creation of default groups) is not needed here - Log.msg("NameServer","store sync database") - for group,smeta,umeta in syncdump[0]: - try: - if group!=':': - dirnam = self.translate(group) - os.mkdir(dirnam) - if smeta: - self._setSystemMeta(group,smeta) - if umeta: - self.setMeta(group,umeta) - except EnvironmentError,x: - Log.warn("NameServer","problem creating group",group,x) - for name,uri,smeta,umeta in syncdump[1]: - try: - origname,name=name,self.validateName(name) - fn=self.translate(name) - open(fn,'w').write(uri+'\n') - if smeta: - self._setSystemMeta(name,smeta) - if umeta: - self.setMeta(name,umeta) - except Pyro.errors.NamingError,x: - Log.warn("NameServer","problem registering name",name,x) - # reset registration of self - try: - self.unregister(Pyro.constants.NAMESERVER_NAME) - except: - pass - self.register(Pyro.constants.NAMESERVER_NAME,oldNSreg) - self.otherNS=twinProxy - Log.msg("NameServer","database sync complete.") - print "Database synchronized." - - def register(self,name,URI): - origname,name=name,self.validateName(name) - URI=self.validateURI(URI) - fn=self.translate(name) - with self.lock: - if os.access(fn,os.R_OK): - Log.msg('NameServer','name already exists:',name) - raise Pyro.errors.NamingError('name already exists',name) - try: - open(fn,'w').write(str(URI)+'\n') - self._dosynccall("register",origname,URI) - Log.msg('NameServer','registered',name,'with URI',str(URI)) - except IOError,x: - if x.errno==errno.ENOENT: - raise Pyro.errors.NamingError('(parent)group not found') - elif x.errno==errno.ENOTDIR: - raise Pyro.errors.NamingError('parent is no group') - else: - raise Pyro.errors.NamingError(str(x)) - - def unregister(self,name): - origname,name=name,self.validateName(name) - fn=self.translate(name) - with self.lock: - try: - os.remove(fn) - self._dosynccall("unregister",origname) - Log.msg('NameServer','unregistered',name) - except OSError,x: - if x.errno==errno.ENOENT: - raise Pyro.errors.NamingError('name not found',name) - elif x.errno==errno.EISDIR: - Log.msg('NameServer','attempt to remove a group:',name) - raise Pyro.errors.NamingError('is a group, not an object',name) - else: - raise Pyro.errors.NamingError(str(x)) - - def resolve(self,name): - # not thread-locked: higher performance and not necessary. - name=self.validateName(name) - fn = self.translate(name) - try: - return Pyro.core.PyroURI(open(fn).read()) - except IOError,x: - if x.errno==errno.ENOENT: - raise Pyro.errors.NamingError('name not found',name) - elif x.errno==errno.EISDIR: - Log.msg('NameServer','attempt to resolve groupname:',name) - raise Pyro.errors.NamingError('attempt to resolve groupname',name) - else: - raise Pyro.errors.NamingError(str(x)) - - def flatlist(self): - dbroot=self.translate(':') - with self.lock: - flat=[] - for f in self._filelist(dbroot,dbroot): - f=self._unescapefilename(f) - flat.append((f, self.resolve(f))) - return flat - - # --- hierarchical naming support - def createGroup(self,groupname): - groupname=self.validateName(groupname) - dirnam = self.translate(groupname) - with self.lock: - try: - os.mkdir(dirnam) - self._dosynccall("createGroup",groupname) - Log.msg('NameServer','created group',groupname) - except OSError,x: - if x.errno in (errno.EEXIST, errno.EBUSY): - raise Pyro.errors.NamingError('group already exists',groupname) - elif x.errno == errno.ENOENT: - raise Pyro.errors.NamingError('(parent)group not found') - else: - raise Pyro.errors.NamingError(str(x)) - - def deleteGroup(self,groupname): - groupname=self.validateName(groupname) - if groupname==':': - Log.msg('NameServer','attempt to deleteGroup root group') - raise Pyro.errors.NamingError('not allowed to delete root group') - dirnam = self.translate(groupname) - with self.lock: - if not os.access(dirnam,os.R_OK): - raise Pyro.errors.NamingError('group not found',groupname) - try: - shutil.rmtree(dirnam) - self._dosynccall("deleteGroup",groupname) - Log.msg('NameServer','deleted group',groupname) - except OSError,x: - if x.errno==errno.ENOENT: - raise Pyro.errors.NamingError('group not found',groupname) - elif x.errno==errno.ENOTDIR: - raise Pyro.errors.NamingError('is no group',groupname) - else: - raise Pyro.errors.NamingError(str(x)) - - def list(self,groupname): - if not groupname: - groupname=':' - groupname=self.validateName(groupname) - dirnam=self.translate(groupname) - with self.lock: - if os.access(dirnam,os.R_OK): - if os.path.isfile(dirnam): - raise Pyro.errors.NamingError('is no group',groupname) - else: - l = dircache.listdir(dirnam) - entries = [] - for e in l: - if e.endswith(_PNS_META_SUFFIX): - continue - else: - objname=self._unescapefilename(e) - if os.path.isdir(os.path.join(dirnam,e)): - entries.append((objname,0)) # dir has code 0 - else: - entries.append((objname,1)) # leaf has code 1 - return entries - raise Pyro.errors.NamingError('group not found',groupname) - - - # --- private methods follow - - def _getSyncDump(self): - def visitor(arg,dirname,names): - shortdirname=dirname[len(self.dbroot)+len(os.path.sep):] - if shortdirname.endswith(_PNS_META_SUFFIX): - return - name = ':'+shortdirname.replace(os.path.sep,'.') - smeta=self._getSystemMeta(name) - umeta=self.getMeta(name) - arg[0].append( (name, smeta,umeta) ) - for n in names: - if n.endswith(_PNS_META_SUFFIX): - continue - n=os.path.join(dirname,n) - if os.path.isfile(n): - v=open(n,'r').read().strip() - name=':'+(n[len(self.dbroot)+len(os.path.sep):]).replace(os.path.sep,'.') - smeta=self._getSystemMeta(name) - umeta=self.getMeta(name) - arg[1].append( (name, v, smeta,umeta) ) - result=( [], [] ) # (groups, names) - os.path.walk(self.dbroot, visitor, result) - return result - - def _unescapefilename(self, name): - parts=name.split('\\') - res=[parts[0]] - myappend=res.append - del parts[0] - for item in parts: - if item[1:2]: - try: - myappend(chr(int(item[:2], 16)) + item[2:]) - except ValueError: - myappend('\\' + item) - else: - myappend('\\' + item) - return "".join(res) - def _escapefilename(self,name): - name=name.replace(os.path.sep,'\\%02x' % ord(os.path.sep)) # escape path separators in the name - name=name.replace(':','\\%02x' % ord(':')) # also get rid of any ':' 's - return name - - # recursive file listing, output is like "find -type f" - # but using NS group separator chars - def _filelist(self,root,path): - try: - (filez,dirz) = Pyro.util.listdir(path) - except OSError: - raise Pyro.errors.NamingError('group not found') - - files=[] - for f in filez: - if f.endswith(_PNS_META_SUFFIX): - continue - elif path==root: - files.append(':'+f) - else: - p=path[len(root):].replace(os.sep, '.') - files.append(':'+p+'.'+f) - for d in dirz: - files.extend(self._filelist(root,os.path.join(path,d))) - return files - - # Pyro NS name to filesystem path translation - def translate(self,name): - if name[0]==':': - name=name[1:] - name=self._escapefilename(name) - args=[self.dbroot]+name.split('.') - return os.path.join(*args) - - def getBranch(self,name): - tr = self.translate(name) - if os.path.exists(tr): - return PersistentNameSpaceNode(filename=tr+_PNS_META_SUFFIX) - else: - raise Pyro.errors.NamingError('name not found',name) - -# XXX this is a bit of a hack. Only metadata is stored here, -# and it's only used from getBranch, which in turn is only used -# from the set/get meta functions. -class PersistentNameSpaceNode(NameSpaceNode): - def __init__(self, filename, name=None, meta=None, owner=None): - NameSpaceNode.__init__(self, name, meta, owner) - self.filename=filename - if not name: - # init from file - try: - (sysmeta, usermeta)=Pyro.util.getPickle().load(open(self.filename,"rb")) - NameSpaceNode.setSystemMeta(self, sysmeta) - NameSpaceNode.setMeta(self, usermeta) - except Exception: - pass # just use empty meta... - else: - self._writeToFile() - def setMeta(self,meta): - NameSpaceNode.setMeta(self, meta) - self._writeToFile() - def setSystemMeta(self,meta): - NameSpaceNode.setSystemMeta(self, meta) - self._writeToFile() - def _writeToFile(self): - Pyro.util.getPickle().dump( (self.getSystemMeta(), self.getMeta()) , open(self.filename,"wb"), Pyro.config.PYRO_PICKLE_FORMAT) - - - -############################################################################# -# -# The broadcast server which listens to broadcast requests of clients who -# want to discover our location, or send other system commands. -# -############################################################################# - - -class BroadcastServer(SocketServer.UDPServer): - - nameServerURI = '' # the Pyro URI of the Name Server - - def __init__(self, addr, bcRequestHandler,norange=0): - if norange: - portrange=1 - else: - portrange=Pyro.config.PYRO_PORT_RANGE - (location,port)=addr - for port in range(port, port+portrange): - try: - SocketServer.UDPServer.__init__(self, (location,port), bcRequestHandler) - return # got it! - except socket.error: - continue # try the next port in the list - raise # port range exhausted... re-raise the socket error. - - def server_activate(self): - self.requestValidator=lambda x,y: 1 # default: accept all - self.shutdown=0 # should the server loop stop? - self.preferredTimeOut=3.0 # preferred timeout for the server loop - - def setNS_URI(self,URI): - self.nameServerURI=str(URI) - def setRequestValidator(self, validator): - self.requestValidator=validator - def keepRunning(self, keep): - self.ignoreShutdown = keep # ignore shutdown requests (i.e. keep running?) - - def bcCallback(self,ins): - for i in ins: - i.handle_request() - - def verify_request(self, req, addr): - return self.requestValidator(req, addr) - - def getServerSocket(self): - return self.socket - - -class bcRequestHandler(SocketServer.BaseRequestHandler): - def handle(self): - Log.msg('BroadcastServer','incoming request from',str(self.client_address[0])) - # request is a simple string - cmd = self.request[0] - if cmd==NS_SYSCMD_LOCATION: - # somebody wants to know our location, give them our URI - self.request[1].sendto(self.server.nameServerURI,self.client_address) - elif cmd==NS_SYSCMD_SHUTDOWN: - # we should die!? - if self.server.ignoreShutdown: - Log.msg('BroadcastServer','Shutdown ignored.') - self.request[1].sendto('Shutdown request denied',self.client_address) - else: - Log.msg('BroadcastServer','Shutdown received.') - print 'BroadcastServer received shutdown request... will shutdown shortly...' - self.request[1].sendto('Will shut down shortly',self.client_address) - self.server.shutdown=1 - else: - Log.warn('BroadcastServer','Invalid command ignored:',cmd) - -# The default BC request validator... accepts everything -# You must subclass this for your own validators -class BCReqValidator(object): - def __call__(self, req, addr): - (cmd,self.sock)=req - self.addr=addr - if cmd==NS_SYSCMD_LOCATION: - return self.acceptLocationCmd() - elif cmd==NS_SYSCMD_SHUTDOWN: - return self.acceptShutdownCmd() - else: - return 0 - def reply(self,msg): - self.sock.sendto(msg,self.addr) - def acceptLocationCmd(self): - return 1 - def acceptShutdownCmd(self): - return 1 - - -############################################################################# - -class NameServerStarter(object): - def __init__(self, identification=None): - Pyro.core.initServer() - self.persistent=False - self.identification=identification - self.started = Pyro.util.getEventObject() - def start(self, *args, **kwargs): # see _start for allowed arguments - kwargs["startloop"]=1 - self._start( *args, **kwargs ) - def initialize(self, *args, **kwargs): # see _start for allowed arguments - kwargs["startloop"]=0 - self._start( *args, **kwargs ) - def getServerSockets(self): - result=self.daemon.getServerSockets() - if self.bcserver: - result.append(self.bcserver.getServerSocket()) - return result - def waitUntilStarted(self,timeout=None): - self.started.wait(timeout) - return self.started.isSet() - def _start(self,hostname=None, nsport=None, bcport=0, keep=0, persistent=0, dbdir=None, Guards=(None,None), allowmultiple=0, dontlookupother=0, verbose=0, startloop=1, role=(Pyro.constants.NSROLE_SINGLE,None), bcaddr=None, nobroadcast=False ): - if nsport is None: - if role[0]==Pyro.constants.NSROLE_SECONDARY: - nsport=Pyro.config.PYRO_NS2_PORT - else: - nsport=Pyro.config.PYRO_NS_PORT - if not bcport: - if role[0]==Pyro.constants.NSROLE_SECONDARY: - bcport=Pyro.config.PYRO_NS2_BC_PORT - else: - bcport=Pyro.config.PYRO_NS_BC_PORT - if not bcaddr: - if role[0]==Pyro.constants.NSROLE_SECONDARY: - bcaddr=Pyro.config.PYRO_NS2_BC_ADDR - else: - bcaddr=Pyro.config.PYRO_NS_BC_ADDR - otherNSuri=None - - try: - if not dontlookupother: - retries=Pyro.config.PYRO_BC_RETRIES - timeout=Pyro.config.PYRO_BC_TIMEOUT - Pyro.config.PYRO_BC_RETRIES=1 - Pyro.config.PYRO_BC_TIMEOUT=0.7 - try: - otherNSuri=NameServerLocator().detectNS(bcaddr=bcaddr) - except Pyro.errors.PyroError: - pass - else: - print 'The Name Server appears to be already running on this segment.' - print '(host:',otherNSuri.address,' port:',otherNSuri.port,')' - if allowmultiple: - print 'WARNING: starting another Name Server in the same segment!' - elif role[0] in (Pyro.constants.NSROLE_PRIMARY, Pyro.constants.NSROLE_SECONDARY): - pass - else: - msg='Cannot start multiple Name Servers in the same network segment.' - print msg - raise Pyro.errors.NamingError(msg) - - if role[0]!=Pyro.constants.NSROLE_SINGLE: - print "Locating twin NameServer." - # Do this before starting our own daemon, otherwise possible deadlock! - # This step is done here to make pretty certain that one of both name - # servers finds the other either *now*, or else later on (below). - # If we omit this step here, deadlock may occur on the attempt below! - otherNS = self.locateTwinNS(role, otherNSuri) - if otherNS: - print "Found twin NameServer at",otherNS.URI.address,"port",otherNS.URI.port - role=(role[0], otherNS) - - Pyro.config.PYRO_BC_RETRIES=retries - Pyro.config.PYRO_BC_TIMEOUT=timeout - daemon = Pyro.core.Daemon(host=hostname, port=nsport,norange=1) - except Pyro.errors.DaemonError,x: - print 'The Name Server appears to be already running on this host.' - print '(or somebody else occupies our port,',nsport,')' - if hostname: - print 'It could also be that the address \''+hostname+'\' is not correct.' - print 'Name Server was not started!' - raise - - if self.identification: - daemon.setAllowedIdentifications([self.identification]) - print 'Requiring connection authentication.' - if Guards[0]: - daemon.setNewConnectionValidator(Guards[0]) - - if persistent: - ns=PersistentNameServer(dbdir,role=role[0], identification=self.identification) - daemon.useNameServer(ns) - NS_URI=daemon.connectPersistent(ns,Pyro.constants.NAMESERVER_NAME) - self.persistent=True - else: - ns=NameServer(role=role[0], identification=self.identification) - daemon.useNameServer(ns) - NS_URI=daemon.connect(ns,Pyro.constants.NAMESERVER_NAME) - self.persistent=False - - self.bcserver=None - if nobroadcast: - Log.msg('NS daemon','Not starting broadcast server due to config option') - if verbose: - print "Not starting broadcast server." - else: - # Try to start the broadcast server. Binding on the magic "" - # address should work, but on some systems (windows) it doesn't. - # Therefore we first try "", if that fails, try "". - # If any address override is in place, use that ofcourse. - notStartedError="" - msg = daemon.validateHostnameAndIP() - if msg: - Log.msg('NS daemon','Not starting broadcast server because of issue with daemon IP address.') - if verbose: - print "Not starting broadcast server." - else: - if bcaddr: - broadcastAddresses=[bcaddr] - else: - broadcastAddresses=["", "", "255.255.255.255"] - for bc_bind in broadcastAddresses: - try: - self.bcserver = BroadcastServer((bc_bind,bcport),bcRequestHandler,norange=1) - break - except socket.error,x: - notStartedError += str(x)+" " - if not self.bcserver: - print 'Cannot start broadcast server. Is somebody else occupying our broadcast port?' - print 'The error(s) were:',notStartedError - print '\nName Server was not started!' - raise Pyro.errors.NamingError("cannot start broadcast server") - - if Guards[1]: - self.bcserver.setRequestValidator(Guards[1]) - self.bcserver.keepRunning(keep) - - if keep: - ns.ignoreShutdown=True - if verbose: - print 'Will ignore shutdown requests.' - else: - ns.ignoreShutdown=False - if verbose: - print 'Will accept shutdown requests.' - - print 'Name server listening on:',daemon.sock.getsockname() - if self.bcserver: - print 'Broadcast server listening on:',self.bcserver.socket.getsockname() - message = daemon.validateHostnameAndIP() - if message: - print "\nWARNING:",message,"\n" - - if Guards[0] or Guards[1]: - if verbose: - print 'Using security plugins:' - if Guards[0]: - clazz=Guards[0].__class__ - if verbose: - print ' NS new conn validator =',clazz.__name__,'from', clazz.__module__, ' ['+sys.modules.get(clazz.__module__).__file__+']' - elif verbose: print ' default NS new conn validator' - if Guards[1]: - clazz=Guards[1].__class__ - if verbose: - print ' BC request validator =',clazz.__name__,'from', clazz.__module__, ' ['+sys.modules.get(clazz.__module__).__file__+']' - elif verbose: print ' default BC request validator' - - ns.publishURI(NS_URI,verbose) - - if self.bcserver: - self.bcserver.setNS_URI(NS_URI) - Log.msg('NS daemon','This is the Pyro Name Server.') - if persistent: - Log.msg('NS daemon','Persistent mode, database is in',ns.getDBDir()) - if verbose: - print 'Persistent mode, database is in',ns.getDBDir() - Log.msg('NS daemon','Starting on',daemon.hostname,'port', daemon.port) - if self.bcserver: - Log.msg('NS daemon','Broadcast server on port',bcport) - else: - Log.msg('NS daemon','No Broadcast server') - - if role[0]==Pyro.constants.NSROLE_PRIMARY: - print "Primary", - elif role[0]==Pyro.constants.NSROLE_SECONDARY: - print "Secondary", - print 'Name Server started.' - - # If we run in primary or secondary mode, resynchronize - # the NS database with the other name server. - # Try again to look it up if it wasn't found before. - - if role[0]!=Pyro.constants.NSROLE_SINGLE: - if not otherNS: - # try again to contact the other name server - print "Locating twin NameServer again." - otherNS = self.locateTwinNS(role, otherNSuri) - role=(role[0], otherNS) - if otherNS: - # finally got it, resync! - print "Found twin NameServer at",otherNS.URI.address,"port",otherNS.URI.port - ns._initialResyncWithTwin(otherNS) - - self.started.set() # signal that we've started (for external threads) - - self.daemon=daemon - if os.name!="java": - daemon.setTimeout(20) - - if startloop: - # I use a timeout here otherwise you can't break gracefully on Windoze - try: - if self.bcserver: - daemon.requestLoop(lambda s=self: not s.bcserver.shutdown, - self.bcserver.preferredTimeOut,[self.bcserver],self.bcserver.bcCallback) - if self.bcserver.shutdown: - self.shutdown(ns) - else: - daemon.requestLoop() - except KeyboardInterrupt: - Log.warn('NS daemon','shutdown on user break signal') - print 'Shutting down on user break signal.' - self.shutdown(ns) - except: - try: - (exc_type, exc_value, exc_trb) = sys.exc_info() - out = ''.join(traceback.format_exception(exc_type, exc_value, exc_trb)[-5:]) - Log.error('NS daemon', 'Unexpected exception, type',exc_type, - '\n--- partial traceback of this exception follows:\n', - out,'\n--- end of traceback') - print '*** Exception occured!!! Partial traceback:' - print out - print '*** Resuming operations...' - finally: - del exc_type, exc_value, exc_trb # delete frame refs to allow proper GC - - Log.msg('NS daemon','Shut down gracefully.') - print 'Name Server gracefully stopped.' - - - def locateTwinNS(self, role, otherNSuri): - try: - retries=Pyro.config.PYRO_BC_RETRIES - timeout=Pyro.config.PYRO_BC_TIMEOUT - Pyro.config.PYRO_BC_RETRIES=1 - Pyro.config.PYRO_BC_TIMEOUT=1 - try: - if role[1]: - (host,port)=(role[1]+':').split(':')[:2] - if len(port)==0: - port=None - else: - port=int(port) - otherNS=NameServerLocator(self.identification).getNS(host,port,trace=0) - else: - if otherNSuri: - otherNS=NameServerLocator(self.identification).getNS(host=otherNSuri.address, port=otherNSuri.port, trace=0) - else: - if role[0]==Pyro.constants.NSROLE_PRIMARY: - port=Pyro.config.PYRO_NS2_BC_PORT - else: - port=Pyro.config.PYRO_NS_BC_PORT - otherNS=NameServerLocator(self.identification).getNS(host=None,port=port,trace=0) - Log.msg("NameServerStarted","Found twin NS at",otherNS.URI) - return otherNS - except Pyro.errors.ConnectionDeniedError,x: - raise - except Exception,x: - print "WARNING: Cannot find twin NS yet: ",x - Log.msg("NameServerStarter","Cannot find twin NS yet:",x) - return None - finally: - Pyro.config.PYRO_BC_RETRIES=retries - Pyro.config.PYRO_BC_TIMEOUT=timeout - - - def handleRequests(self,timeout=None): - # this method must be called from a custom event loop - if self.bcserver: - self.daemon.handleRequests(timeout, [self.bcserver], self.bcserver.bcCallback) - if self.bcserver.shutdown: - self.shutdown() - else: - self.daemon.handleRequests(timeout) - - def shutdown(self, ns=None): - if ns: - # internal shutdown call with specified NS object - daemon=ns.getDaemon() - else: - # custom shutdown call w/o specified NS object, use stored instance - daemon=self.daemon - ns=daemon.getNameServer() - del self.daemon - ns._removeTwinNS() - if not self.persistent: - daemon.disconnect(ns) # clean up nicely only if not running in persistent mode - if self.bcserver: - self.bcserver.shutdown=1 - daemon.shutdown() - -def main(argv): - Args = Pyro.util.ArgParser() - Args.parse(argv,'hkmrvxn:p:b:c:d:s:i:1:2:') - if Args.hasOpt('h'): - print 'Usage: pyro-ns [-h] [-k] [-m] [-r] [-x] [-n hostname] [-p port] [-b bcport] [-c bcaddr]' - print ' [-i identification] [-d [databaselocation]] [-s securitymodule]' - print ' [-1 [host:port]] [-2 [host:port]] [-v]' - print ' where -p = NS server port (0 for auto)' - print ' -n = non-default hostname to bind on' - print ' -b = NS broadcast port' - print ' -c = NS broadcast address override' - print ' -x = do not start a broadcast listener' - print ' -m = allow multiple instances in network segment' - print ' -r = don\'t attempt to find already existing nameservers' - print ' -k = keep running- do not respond to shutdown requests' - print ' -d = use persistent database, provide optional storage directory' - print ' -s = use given python module with security plugins' - print ' -i = specify the required authentication ID' - print ' -1 = runs this NS as primary, opt. specify where secondary is' - print ' -2 = runs this NS as secondary, opt. specify where primary is' - print ' -v = verbose output' - print ' -h = print this help' - raise SystemExit - host = Args.getOpt('n',None) - port = Args.getOpt('p',None) - if port: - port=int(port) - bcport = int(Args.getOpt('b',0)) - bcaddr = Args.getOpt('c',None) - nobroadcast = Args.hasOpt('x') - - role=Pyro.constants.NSROLE_SINGLE - roleArgs=None - if Args.hasOpt('1'): - role=Pyro.constants.NSROLE_PRIMARY - roleArgs=Args.getOpt('1') - if Args.hasOpt('2'): - role=Pyro.constants.NSROLE_SECONDARY - roleArgs=Args.getOpt('2') - - ident = Args.getOpt('i',None) - verbose = Args.hasOpt('v') - keep=Args.hasOpt('k') - allowmultiple=Args.hasOpt('m') - dontlookupother=Args.hasOpt('r') - - try: - dbdir = Args.getOpt('d') - persistent = 1 - except KeyError: - persistent = 0 - dbdir = None - - try: - secmod = __import__(Args.getOpt('s'),locals(),globals()) - Guards = (secmod.NSGuard(), secmod.BCGuard()) - except ImportError,x: - print 'Error loading security module:',x - print '(is it in your python import path?)' - raise SystemExit - except KeyError: - secmod = None - Guards = (None,None) - - Args.printIgnored() - if Args.args: - print 'Ignored arguments:', ' '.join(Args.args) - - print '*** Pyro Name Server ***' - if ident: - starter=NameServerStarter(identification=ident) - else: - starter=NameServerStarter() - - try: - starter.start(host,port,bcport,keep,persistent,dbdir,Guards,allowmultiple,dontlookupother,verbose,role=(role,roleArgs),bcaddr=bcaddr,nobroadcast=nobroadcast) - except (Pyro.errors.NamingError, Pyro.errors.DaemonError),x: - # this error has already been printed, just exit. - pass - - -# allow easy starting of the NS by using python -m -if __name__=="__main__": - main(sys.argv[1:]) diff --git a/lib/Pyro/nsc.py b/lib/Pyro/nsc.py deleted file mode 100644 index 759e0f66998..00000000000 --- a/lib/Pyro/nsc.py +++ /dev/null @@ -1,231 +0,0 @@ -############################################################################# -# -# Pyro Name Server Control Tool -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -import Pyro.constants -import Pyro.util -import Pyro.core -import Pyro.errors -from Pyro.naming import NameServerLocator -from Pyro.errors import NamingError, ConnectionDeniedError, PyroError -from Pyro.protocol import getHostname - -class PyroNSControl(object): - def args(self, args): - self.Args = Pyro.util.ArgParser() - self.Args.parse(args,'h:p:c:i:') - self.Args.printIgnored() - if self.Args.args: - cmd = self.Args.args[0] - del self.Args.args[0] - return cmd - return None - - def connect(self, sysCmd=None): - host = self.Args.getOpt('h',None) - bcaddr = self.Args.getOpt('c',None) - port = int(self.Args.getOpt('p', 0)) - ident = self.Args.getOpt('i',None) - if port==0: - port=None - - locator = NameServerLocator(identification=ident) - if not sysCmd: - self.NS = locator.getNS(host,port,1,bcaddr=bcaddr) - print 'NS is at',self.NS.URI.address,'('+(getHostname(self.NS.URI.address) or '??')+') port',self.NS.URI.port - self.NS._setIdentification(ident) - else: - result = locator.sendSysCommand(sysCmd,host,port,1,bcaddr=bcaddr) - print 'Result from system command',sysCmd,':',result - - def handleError(self, msg, exc): - print "## %s: " % msg, - if isinstance(exc.args, (list, tuple)): - print "; ".join(exc.args[:-1]), - else: - print exc.args, - print " ##" - - def ping(self): - self.connect() - self.NS.ping() - print 'NS is up and running!' - - def listall(self): - self.connect() - flat=self.NS.flatlist() - flat.sort() - print '-------------- START DATABASE' - for (name,val) in flat: - print name,' --> ',str(val) - print '-------------- END' - - def list(self): - self.connect() - if not self.Args.args: - # list the current group - print self.NS.fullName(''),'-->', - self.printList(self.NS.list(None)) - else: - # list all subpaths - for n in self.Args.args: - print self.NS.fullName(n),' -->', - try: - self.printList(self.NS.list(n)) - except NamingError,x: - self.handleError("can't list", x) - - def printList(self,list): - list.sort() - print '(', - for (n,t) in list: - if t==0: - print '['+n+']', - elif t==1: - print n, - print ')' - - def resolve(self): - self.connect() - if not self.Args.args: - print 'No arguments, nothing to resolve' - else: - for n in self.Args.args: - print n,' -->', - try: - print self.NS.resolve(n) - except NamingError,x: - self.handleError("can't resolve", x) - - def register(self): - self.connect() - try: - self.NS.register(self.Args.args[0],self.Args.args[1]) - uri=Pyro.core.PyroURI(self.Args.args[1]) - print 'registered',self.Args.args[0],' --> ',uri - except NamingError,x: - self.handleError('Error from NS',x) - except IndexError: - print 'Register needs 2 args: name URI' - - def remove(self): - self.connect() - for n in self.Args.args: - try: - self.NS.unregister(n) - print n,'unregistered.' - except NamingError,x: - self.handleError("Can't unregister", x) - - def creategroup(self): - self.connect() - for n in self.Args.args: - try: - self.NS.createGroup(n) - print n,'created.' - except NamingError,x: - self.handleError("Can't create group '"+n+"'",x) - - def deletegroup(self): - self.connect() - for n in self.Args.args: - try: - self.NS.deleteGroup(n) - print n,'deleted.' - except NamingError,x: - self.handleError("Can't delete group '"+n+"'",x) - - def showmeta(self): - self.connect() - if not self.Args.args: - print 'No arguments, nothing to show meta of' - for n in self.Args.args: - try: - print "META INFO OF",self.NS.fullName(n) - print "system meta info :",self.NS._getSystemMeta(n) - print " user meta info :",self.NS.getMeta(n) - except NamingError,x: - self.handleError("Can't get metadata",x) - - def setmeta(self): - self.connect() - try: - if len(self.Args.args)>2: - raise IndexError - name=self.Args.args[0] - meta=self.Args.args[1] - self.NS.setMeta(name,meta) - print "Metadata of",name,"set." - except IndexError: - print 'Setmeta needs 2 args: name metadata' - - def resync(self): - self.connect() - self.NS.resync() - print 'resync done' - - def shutdown(self): - self.connect(sysCmd='shutdown') - - -def usage(): - print 'PyroNS control program - usage is as follows;' - print '>> pyro-nsc [-h host] [-p port] [-c bcaddr] [-i identification] command [args...]' - print 'where command is one of: ping, list, listall, resolve, register, remove, creategroup, deletegroup, showmeta, setmeta, resync, shutdown' - print ' host is the host where the NS should be contacted' - print ' port is the non-standard Pyro NS broadcast port' - print ' (if host is specified, it is the Pyro port instead)' - print ' bcaddr allows you to override the broadcast address' - print ' identification is the authentication ID to connect to the server' - print ' args... depend on the command.' - raise SystemExit - -def main(argv): - - ctrl = PyroNSControl() - cmd=ctrl.args(argv) - - if not cmd: - usage() - - try: - # nice construct to map commands to the member function to call - call= { 'ping': ctrl.ping, - 'list': ctrl.list, - 'listall': ctrl.listall, - 'resolve': ctrl.resolve, - 'register': ctrl.register, - 'remove': ctrl.remove, - 'creategroup': ctrl.creategroup, - 'deletegroup': ctrl.deletegroup, - 'shutdown': ctrl.shutdown, - 'showmeta': ctrl.showmeta, - 'setmeta': ctrl.setmeta, - 'resync': ctrl.resync } [cmd] - - except KeyError: - usage() - try: - Pyro.core.initClient(banner=0) - call() - except ConnectionDeniedError,arg: - print 'Could not connect to the server:',arg - if str(arg)==Pyro.constants.deniedReasons[Pyro.constants.DENIED_SECURITY]: - print "Supply correct authentication ID?" - except PyroError,arg: - print 'There is a problem:',arg - except Exception,x: - print 'CAUGHT ERROR, printing Pyro traceback >>>>>>',x - print ''.join(Pyro.util.getPyroTraceback(x)) - print '<<<<<<< end of Pyro traceback' - - -# allow easy usage with python -m -if __name__=="__main__": - import sys - main(sys.argv[1:]) diff --git a/lib/Pyro/protocol.py b/lib/Pyro/protocol.py deleted file mode 100644 index 4d2e49980d0..00000000000 --- a/lib/Pyro/protocol.py +++ /dev/null @@ -1,1291 +0,0 @@ -############################################################################# -# -# Pyro Protocol Adapters -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -from __future__ import with_statement -import socket, struct, os, time, sys, hmac, types, random, errno, select -import imp, marshal, new, __builtin__ -try: - import hashlib - md5=hashlib.md5 -except ImportError: - import md5 - md5=md5.md5 -import Pyro.constants, Pyro.util -from Pyro.util import HostUtil - -from Pyro.errors import * -from Pyro.errors import _InternalNoModuleError -pickle = Pyro.util.getPickle() -Log = Pyro.util.Log - -if Pyro.util.supports_multithreading(): - from threading import Thread,currentThread - _has_threading = 1 -else: - _has_threading = 0 - -if Pyro.util.supports_compression(): - import zlib - _has_compression = 1 -else: - _has_compression = 0 - - -try: - from M2Crypto import SSL - from M2Crypto.SSL import SSLError - if _has_threading: - import M2Crypto - M2Crypto.threading.init() -except ImportError: - class SSLError(Exception): pass - -ERRNO_RETRIES=[errno.EINTR, errno.EAGAIN, errno.EWOULDBLOCK, errno.EINPROGRESS] -if hasattr(errno, "WSAEINTR"): - ERRNO_RETRIES.append(errno.WSAEINTR) -if hasattr(errno, "WSAEWOULDBLOCK"): - ERRNO_RETRIES.append(errno.WSAEWOULDBLOCK) -if hasattr(errno, "WSAEINPROGRESS"): - ERRNO_RETRIES.append(errno.WSAEINPROGRESS) - -#------ Get the hostname (possibly of other machines) (returns None on error) -def getHostname(ip=None): - try: - if ip: - hn,alias, ips = HostUtil.get_inst().gethostbyaddr(ip) - return hn - else: - return HostUtil.get_inst().gethostname() - except socket.error: - return None - -#------ Get IP address (return None on error) -def getIPAddress(host=None): - try: - return HostUtil.get_inst().gethostbyname(host or getHostname()) - except socket.error: - return None - - -#------ Socket helper functions for sending and receiving data correctly. - - -# process optional timeout on socket. -# notice the check for M2Crypto SSL sockets: if there's data pending, -# a select on them will fail. So we avoid calling select in that case. -def _sock_timeout_send(sock, timeout): - if timeout and (not hasattr(sock,'pending') or sock.pending()==0): - r,w,e=safe_select([],[sock],[],timeout) - if not w: - raise TimeoutError('connection timeout sending') - -def _sock_timeout_recv(sock, timeout): - if timeout and (not hasattr(sock,'pending') or sock.pending()==0): - r,w,e=safe_select([sock],[],[],timeout) - if not r: - raise TimeoutError('connection timeout receiving') - -# Receive a precise number of bytes from a socket. Raises the -# ConnectionClosedError if that number of bytes was not available. -# (the connection has probably been closed then). -# Never will this function return an empty message (if size>0). -# We need this because 'recv' isn't guaranteed to return all desired -# bytes in one call, for instance, when network load is high. -# Use a list of all chunks and join at the end: faster! -# Handle EINTR states (interrupted system call) by just retrying. -def sock_recvmsg(sock, size, timeout=0): - while True: - try: - return _recv_msg(sock,size,timeout) - except socket.timeout: - raise TimeoutError("connection timeout receiving") - except socket.error,x: - if x.args[0] == errno.EINTR or (hasattr(errno, 'WSAEINTR') and x.args[0] == errno.WSAEINTR): - # interrupted system call, just retry - continue - raise ConnectionClosedError('connection lost: %s' % x) - except SSLError,x: - raise ConnectionClosedError('connection lost: %s' % x) - -# select the optimal recv() implementation -if hasattr(socket,"MSG_WAITALL") and not Pyro.config.PYRO_BROKEN_MSGWAITALL: - def _recv_msg(sock,size,timeout): - _sock_timeout_recv(sock,timeout) - try: - chunk=sock.recv(size, socket.MSG_WAITALL) # receive all data in one call - except TypeError: - # M2Crypto sock.recv() doesn't support MSG_WAITALL parameter - return __recv_msg_compat(sock,size,timeout) - else: - if len(chunk)!=size: - err=ConnectionClosedError('connection lost') - err.partialMsg=chunk # store the message that was received until now - raise err - return chunk -else: - def _recv_msg(sock,size,timeout): - _sock_timeout_recv(sock, timeout) - return __recv_msg_compat(sock,size,timeout) - -def __recv_msg_compat(sock,size,timeout): # compatibility implementation for non-MSG_WAITALL / M2Crypto - msglen=0 - msglist=[] - # Receive chunks of max. 60kb size: - # (rather arbitrary limit, but it avoids memory/buffer problems on certain OSes -- VAX/VMS, Windows) - while msglen", "exec") - else: - code = marshal.loads(module[8:]) - - importer=None - try: - loaded = 0 - # XXX probably want maxtries here... - while not loaded: - # install a custom importer to intercept any extra needed modules - # when executing the module code just obtained from the server - imp.acquire_lock() - importer = agent_import(__builtin__.__import__) - __builtin__.__import__ = importer - imp.release_lock() - - try: - exec code in mod.__dict__ - loaded = 1 - except ImportError: - mname = importer.name - if importer is not None: - __builtin__.__import__ = importer.orig_import - importer = None - - # XXX probably want maxrecursion here... - self._retrieveCode(mname, level+1) - - finally: - if importer is not None: - __builtin__.__import__ = importer.orig_import - finally: - imp.release_lock() # release the global import lock - - - def _remoteInvocationMobileCode(self, method, flags, *args): - # special trimmed-down version for mobile code methods (no locking etc) - body=pickle.dumps((self.URI.objectID,method,flags,args),Pyro.config.PYRO_PICKLE_FORMAT) - sock_sendmsg(self.conn.sock, self.createMsg(body), self.timeout) - ver,answer,pflags = self.receiveMsg(self.conn,1) - if answer is None: - raise ProtocolError('incorrect answer received') - answer=pickle.loads(answer) - if isinstance(answer,PyroExceptionCapsule): - if isinstance(answer.excObj,_InternalNoModuleError): - # server couldn't load module, supply it - return self.processMissingModuleError(answer.excObj, method, flags, args) - else: - # we have an encapsulated exception, raise it again. - answer.raiseEx() - return answer - - def remoteInvocation(self, method, flags, *args): - with self.lock: - # only 1 thread at a time may use this connection to call a remote method - try: - self.__pyrocallbusy=True - return self._remoteInvocation(method, flags, *args) - self.__pyrocallbusy=False - finally: - if self.__pyrocallbusy: - # the call has been aborted before completion, close the connection - # to avoid corrupt transfers on the next call - self.release() - - def _remoteInvocation(self, method, flags, *args): - if 'conn' not in self.__dict__.keys(): - Log.error('PYROAdapter','no connection available in remoteinvocation') - raise ProtocolError('no connection available in remoteinvocation') - if method in self.onewayMethods: - flags |= Pyro.constants.RIF_Oneway - body=pickle.dumps((self.URI.objectID,method,flags,args),Pyro.config.PYRO_PICKLE_FORMAT) - try: - sock_sendmsg(self.conn.sock, self.createMsg(body), self.timeout) - except (socket.error, ProtocolError, KeyboardInterrupt): - # Communication error during write. To avoid corrupt transfers, we close the connection. - # Otherwise we might receive the previous reply as a result of a new methodcall! - # Special case for keyboardinterrupt: people pressing ^C to abort the client - # may be catching the keyboardinterrupt in their code. We should probably be on the - # safe side and release the proxy connection in this case too, because they might - # be reusing the proxy object after catching the exception... - self.release() - raise - else: - if flags & Pyro.constants.RIF_Oneway: - self.__pyrocallbusy=False - return None # no answer required, return immediately - ver,answer,pflags = self.receiveMsg(self.conn,1) # read the server's response, send no further replies - self.__pyrocallbusy=False - if answer is None: - raise ProtocolError('incorrect answer received') - - # Try to get the answer from the server. - # If there are import problems, try to get those modules from - # the server too (if mobile code is enabled). - if not Pyro.config.PYRO_MOBILE_CODE: - answer = pickle.loads(answer) - else: - importer=None - try: - imp.acquire_lock() - loaded = 0 - # XXX maxtries here... - while not loaded: - # install a custom importer to intercept any extra needed modules - # when unpickling the answer just obtained from the server - imp.acquire_lock() - importer = agent_import(__builtin__.__import__) - __builtin__.__import__ = importer - imp.release_lock() - - try: - answer = pickle.loads(answer) - loaded = 1 - except ImportError: - mname = importer.name - if importer is not None: - __builtin__.__import__ = importer.orig_import - importer = None - self._retrieveCode(mname, 0) - - finally: - if importer is not None: - __builtin__.__import__ = importer.orig_import - imp.release_lock() - - if isinstance(answer,PyroExceptionCapsule): - if isinstance(answer.excObj,_InternalNoModuleError): - # server couldn't load the module, send it - return self.processMissingModuleError(answer.excObj, method, flags, args) - else: - # we have an encapsulated exception, raise it again. - answer.raiseEx() - return answer - - def processMissingModuleError(self, errorinfo, method, flags, args): - # server couldn't load module, supply it - # XXX this code is ugly. and duplicated in remote_retrieve_code in core.py - Log.msg('PYROAdapter',"server can't load module: "+errorinfo.modulename) - try: - importmodule=new.module('-agent-import-') - mname=errorinfo.modulename - # not used: fromlist=errorinfo.fromlist - try: - exec 'import '+mname in importmodule.__dict__ - except ImportError: - Log.error('PYROAdapter','Server wanted a non-existing module:',mname) - raise PyroError('Server wanted a non-existing module',mname) - m=eval('importmodule.'+mname) - bytecode=None - if hasattr(m,"_PYRO_bytecode"): - # use the bytecode that was put there earlier, - # this avoids recompiles of the source .py if we don't have .pyc bytecode available - bytecode=m._PYRO_bytecode - else: - # try to load the module's compiled source, or the real .py source if that fails. - # note that the source code (.py) is opened with universal newline mode - if not hasattr(m,"__file__"): - raise PyroError("cannot read module source code",mname) - (filebase,ext)=os.path.splitext(m.__file__) - if ext.startswith(".PY"): - exts = ( (".PYO","rb"), (".PYC","rb"), (".PY","rU") ) # uppercase - else: - exts = ( (".pyo","rb"), (".pyc","rb"), (".py","rU") ) # lowercase - for ext,mode in exts: - try: - bytecode=open(filebase+ext, mode).read() - break - except EnvironmentError: - pass - if bytecode: - Log.msg('PYROAdapter',"sending module to server: "+mname) - self._remoteInvocationMobileCode("remote_supply_code",0,mname, bytecode, self.conn.sock.getsockname()) - # retry the method invocation - return self._remoteInvocation(* (method, flags)+args) # use the non-locking call - Log.error("PYROAdapter","cannot read module source code for module:", mname) - raise PyroError("cannot read module source code",mname) - finally: - del importmodule - - # (private) receives a socket message, returns: (protocolver, message, protocolflags) - def receiveMsg(self,conn,noReply=0): - try: - msg=sock_recvmsg(conn.sock, self.headerSize, self.timeout) - (hid, ver, hsiz, bsiz, pflags, crc) = struct.unpack(self.headerFmt,msg) - # store in the connection what pickle method this is - if pflags&PFLG_XMLPICKLE_GNOSIS: - conn.pflags|=PFLG_XMLPICKLE_GNOSIS - if ver!=self.version: - msg='incompatible protocol version' - Log.error('PYROAdapter',msg) - if not noReply: - # try to report error to client, but most likely the connection will terminate: - self.returnException(conn, ProtocolError(msg)) - raise ProtocolError(msg) - if hid!=self.headerID or hsiz!=self.headerSize: - msg='invalid header' - Log.error('PYROAdapter',msg) - Log.error('PYROAdapter','INVALID HEADER DETAILS: ',conn,( hid, ver, hsiz, bsiz,pflags)) - if not noReply: - # try to report error to client, but most likely the connection will terminate: - self.returnException(conn, ProtocolError(msg), shutdown=1) - raise ProtocolError(msg) - body=sock_recvmsg(conn.sock, bsiz, self.timeout) - if pflags&PFLG_CHECKSUM: - if _has_compression: - if crc!=zlib.adler32(body): - msg='checksum error' - Log.error('PYROAdapter',msg) - if not noReply: - self.returnException(conn, ProtocolError(msg)) - raise ProtocolError(msg) - else: - raise ProtocolError('cannot perform checksum') - if pflags&PFLG_COMPRESSED: - if _has_compression: - body=zlib.decompress(body) - else: - # We received a compressed message but cannot decompress. - # Is this really a server error? We now throw an exception on the server... - raise ProtocolError('compression not supported') - return ver,body,pflags - except (socket.error, ProtocolError, KeyboardInterrupt),x: - # Communication error during read. To avoid corrupt transfers, we close the connection. - # Otherwise we might receive the previous reply as a result of a new methodcall! - # Special case for keyboardinterrupt: people pressing ^C to abort the client - # may be catching the keyboardinterrupt in their code. We should probably be on the - # safe side and release the proxy connection in this case too, because they might - # be reusing the proxy object after catching the exception... - self.release() - raise - - def _unpickleRequest(self, pflags, body): - if pflags&PFLG_XMLPICKLE_GNOSIS: - if Pyro.config.PYRO_XML_PICKLE=='gnosis': - return pickle.loads(body) - else: - return Pyro.util.getXMLPickle('gnosis').loads(body) - elif Pyro.config.PYRO_XML_PICKLE: - Log.error('PYROAdapter','xml pickle required, got other pickle') - raise ProtocolError('xml pickle required, got other pickle') - else: - return pickle.loads(body) - - def handleInvocation(self,daemon,conn): - ver,body,pflags = self.receiveMsg(conn) - if not body: - # something went wrong even before receiving the full message body - return - if ver!=self.version: - Log.error('PYROAdapter','incompatible protocol version') - self.returnException(conn, ProtocolError('incompatible protocol version')) - return - - # Unpickle the request, which is a tuple: - # (object ID, method name, flags, (arg1,arg2,...)) - importer=fromlist=None - try: - if Pyro.config.PYRO_MOBILE_CODE: - # install a custom importer to intercept any extra needed modules - # when unpickling the request just obtained from the client - try: - imp.acquire_lock() - importer=agent_import(__builtin__.__import__) - __builtin__.__import__=importer - req=self._unpickleRequest(pflags, body) - finally: - __builtin__.__import__=importer.orig_import - imp.release_lock() - else: - # no mobile code; just unpickle the stuff without a custom importer. - req=self._unpickleRequest(pflags, body) - - if type(req)!=tuple or len(req)!=4 or type(req[3])!=tuple: - # sanity check failed - raise ProtocolError("invalid request data format") - - except ImportError,x: - if Pyro.config.PYRO_MOBILE_CODE: - # return a special exception that will be processed by client; - # it will call the internal 'remote_supply_code' member - if importer: - modname=importer.name - fromlist=importer.fromlist - else: - modname = x.args[0][16:] - fromlist=None - self.returnException(conn, _InternalNoModuleError(modname,fromlist),0) # don't shutdown! - else: - Log.error('PYROAdapter','code problem with incoming object: '+str(x)) - self.returnException(conn, NoModuleError(* x.args)) - return - - try: - # find the object in the implementation database of our daemon - o=daemon.getLocalObject(req[0]) - except (KeyError, TypeError) ,x: - Log.warn('PYROAdapter','Invocation to unknown object ignored:',x) - self.returnException(conn, ProtocolError('unknown object ID')) - return - else: - # Do the invocation. We are already running in our own thread. - if req[2]&Pyro.constants.RIF_Oneway and Pyro.config.PYRO_ONEWAY_THREADED and daemon.threaded: - # received a oneway call, run this in its own thread. - thread=Thread(target=self._handleInvocation2, args=(daemon,req,pflags,conn,o,True)) - thread.setDaemon(1) # thread must exit at program termination. - thread.localStorage=LocalStorage() # set local storage for the new thread - thread.start() - else: - # not oneway or not in threaded mode, just do the invocation synchronously - self._handleInvocation2(daemon,req,pflags,conn,o,False) - - def _handleInvocation2(self, daemon, req, pflags, conn, obj, mustInitTLS=False): - if mustInitTLS: - daemon.initTLS(daemon.getLocalStorage()) - try: - flags=req[2] - importer=None - if not Pyro.config.PYRO_MOBILE_CODE: - res = obj.Pyro_dyncall(req[1],flags,req[3]) # (method,flags,args) - else: - try: - # install a custom importer to intercept any extra needed modules - # when executing the remote method. (using the data passed in by - # the client may trigger additional imports) - imp.acquire_lock() - importer=agent_import(__builtin__.__import__) - __builtin__.__import__=importer - res = obj.Pyro_dyncall(req[1],flags,req[3]) # (method,flags,args) - finally: - __builtin__.__import__=importer.orig_import - imp.release_lock() - - if flags&Pyro.constants.RIF_Oneway: - return # no result, return immediately - # reply the result to the caller - if pflags&PFLG_XMLPICKLE_GNOSIS: - replyflags=PFLG_XMLPICKLE_GNOSIS - if Pyro.config.PYRO_XML_PICKLE=='gnosis': - body=pickle.dumps(res,Pyro.config.PYRO_PICKLE_FORMAT) - else: - body=Pyro.util.getXMLPickle('gnosis').dumps(res,Pyro.config.PYRO_PICKLE_FORMAT) - else: - replyflags=0 - body=pickle.dumps(res,Pyro.config.PYRO_PICKLE_FORMAT) - sock_sendmsg(conn.sock, self.createMsg(body,replyflags),self.timeout) - except ImportError,ix: - if Pyro.config.PYRO_MOBILE_CODE: - # Return a special exception that will be processed by client; - # it will call the internal 'remote_supply_code' member. - # We have to use this seemingly complex way to signal the client - # to supply us some code, but it is only a proxy! We can't *call* it! - if importer: - # grab the import info from our importer - name=importer.name - fromlist=importer.fromlist - else: - # XXX the importerror sometimes doesn't contain the package :-( - name=ix.args[0][16:] - fromlist=None - Log.msg('PYROAdapter','failed to import',name) - self.returnException(conn, _InternalNoModuleError(name,fromlist),0) # don't shutdown! - else: - Log.error('PYROAdapter','code problem with incoming object: '+str(ix)) - self.returnException(conn, NoModuleError(* ix.args)) - except Exception: - # Handle the exception. Pass in if it was a oneway call, - # those calls don't need any response to be sent. - daemon.handleError(conn, bool(flags&Pyro.constants.RIF_Oneway)) - - def returnException(self, conn, exc, shutdown=1, args=None): - # return an encapsulated exception to the client - if conn.pflags&PFLG_XMLPICKLE_GNOSIS: - pic=Pyro.util.getXMLPickle('gnosis') - else: - pic=pickle - try: - body=pic.dumps(PyroExceptionCapsule(exc,args),Pyro.config.PYRO_PICKLE_FORMAT) - except Exception,x: - # hmm, pickling the exception failed... pickle the string instead - body=pic.dumps(PyroExceptionCapsule(PyroError(str(x)),args),Pyro.config.PYRO_PICKLE_FORMAT) - sock_sendmsg(conn.sock, self.createMsg(body),self.timeout) - if shutdown: - conn.close() - - def handleConnection(self, conn, tcpserver): - # Server-side connection stuff. Use auth code from tcpserver's validator. - try: - # Validate the connection source (host) immediately, - # if it's ok, send authentication challenge, and read identification data to validate. - (ok,reasonCode) = tcpserver.newConnValidator.acceptHost(tcpserver,conn) - if ok: - challenge=tcpserver.newConnValidator.createAuthChallenge(tcpserver,conn) - if len(challenge)!=self.AUTH_CHALLENGE_SIZE: - raise ValueError("Auth challenge must be exactly "+`self.AUTH_CHALLENGE_SIZE`+" bytes") - sock_sendmsg(conn.sock, self.createMsg(challenge),self.timeout) - ver,body,pflags = self.receiveMsg(conn) - # only process the message if it makes a bit of sense - if ver==self.version and body.startswith(self.connectMSG): - token=body[len(self.connectMSG):] - (ok,reasonCode) = tcpserver.newConnValidator.acceptIdentification(tcpserver,conn,token,challenge) - if ok: - self.sendAccept(conn) - conn.connected=1 - return 1 - else: - self.sendDeny(conn,reasonCode) - else: - self.sendDeny(conn,reasonCode) - return 0 - except ProtocolError: - # ignore the message if it caused protocol errors - return 0 - -# import wrapper class to help with importing remote modules -class agent_import(object): - def __init__(self, orig_import): - self.orig_import=orig_import - def __call__(self,name,iglobals={},ilocals={},fromlist=None, *rest, **krest): - if os.name=="java": - # workaround for odd Jython bug, iglobals and ilocals may not exist in this scope...(?!) - iglobals=vars().get("iglobals",{}) - ilocals=vars().get("ilocals",{}) - # save the import details: - self.name=name # note: this must be a str object - self.fromlist=fromlist - return self.orig_import(name,iglobals,ilocals,fromlist, *rest, **krest) - -# -# The SSL adapter that handles SSL connections instead of regular sockets. -# -class PYROSSLAdapter(PYROAdapter): - def __init__(self): - PYROAdapter.__init__(self) - try: - from M2Crypto import SSL - except ImportError: - raise ProtocolError('SSL not available') - - self.ctx = SSL.Context('sslv23') - if Pyro.config.PYROSSL_KEY: - keyfile = os.path.join(Pyro.config.PYROSSL_CERTDIR, Pyro.config.PYROSSL_KEY) - else: - keyfile = None - self.ctx.load_cert(os.path.join(Pyro.config.PYROSSL_CERTDIR, Pyro.config.PYROSSL_CERT), - keyfile) - self.ctx.load_client_ca(os.path.join(Pyro.config.PYROSSL_CERTDIR, Pyro.config.PYROSSL_CA_CERT)) - self.ctx.load_verify_info(os.path.join(Pyro.config.PYROSSL_CERTDIR, Pyro.config.PYROSSL_CA_CERT)) - self.ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert,10) - self.ctx.set_allow_unknown_ca(1) - Log.msg('PYROSSLAdapter','SSL Context initialized') - - def setTimeout(self, timeout): - PYROAdapter.setTimeout(self, timeout) - - def bindToURI(self,URI): - if URI.protocol not in ('PYROSSL','PYROLOCSSL'): - Log.error('PYROSSLAdapter','incompatible protocol in URI:',URI.protocol) - raise ProtocolError('incompatible protocol in URI') - with self.lock: # only 1 thread at a time can bind the URI - try: - self.URI=URI - sock = SSL.Connection(self.ctx,socket.socket(socket.AF_INET, socket.SOCK_STREAM)) - if not Pyro.config.PYROSSL_POSTCONNCHECK: - sock.postConnectionCheck=None - _connect_socket(sock, URI.address, URI.port, self.timeout) - conn=TCPConnection(sock, sock.getpeername()) - # receive the authentication challenge string, and use that to build the actual identification string. - authChallenge=self.recvAuthChallenge(conn) - # reply with our ident token, generated from the ident passphrase and the challenge - msg = self._sendConnect(sock,self.newConnValidator.createAuthToken(self.ident, authChallenge, conn.addr, self.URI, None) ) - if msg==self.acceptMSG: - self.conn=conn - self.conn.connected=1 - Log.msg('PYROSSLAdapter','connected to',str(URI)) - if URI.protocol=='PYROLOCSSL': - self.resolvePYROLOC_URI("PYROSSL") # updates self.URI - elif msg[:len(self.denyMSG)]==self.denyMSG: - try: - raise ConnectionDeniedError(Pyro.constants.deniedReasons[int(msg[-1])]) - except (KeyError,ValueError): - raise ConnectionDeniedError('invalid response') - except socket.error: - Log.msg('PYROSSLAdapter','connection failed to URI',str(URI)) - raise ProtocolError('connection failed') - - def _sendConnect(self, sock, ident): - return PYROAdapter._sendConnect(self, sock, ident) - - -def getProtocolAdapter(protocol): - if protocol in ('PYRO', 'PYROLOC'): - return PYROAdapter() - elif protocol in ('PYROSSL', 'PYROLOCSSL'): - return PYROSSLAdapter() - else: - Log.error('getProtocolAdapter','unsupported protocol:',protocol) - raise ProtocolError('unsupported protocol') - - -#-------- TCPConnection object for TCPServer class -class TCPConnection(object): - def __init__(self, sock, addr): - self.sock = sock - set_sock_keepalive(self.sock) # enable tcp/ip keepalive on this socket - self.addr = addr - self.connected=0 # connected? - self.pflags=0 # protocol flags - def __del__(self): - self.close() - def fileno(self): - return self.sock.fileno() - def close(self): - #self.sock.makefile().flush() - self.sock.close() - self.connected=0 - def shutdown(self): - #self.sock.makefile().flush() - self.sock.shutdown(2) # no further send/receives - def __str__(self): - return 'TCPConnection with '+str(self.addr)+' connected='+str(self.connected) - -#-------- The New Connection Validators: -#-------- DefaultConnValidator checks max number of connections & identification -#-------- and ident check is done using hmac-md5 secure hash of passphrase+challenge. -#-------- Contains client- & server-side auth code. -class DefaultConnValidator(object): - def __init__(self): - self.setAllowedIdentifications(None) # default=accept all (None means all!) - def acceptHost(self,daemon,connection): - if len(daemon.connections)>=Pyro.config.PYRO_MAXCONNECTIONS: - Log.msg('DefaultConnValidator','Too many open connections, closing',connection,'#conns=',len(daemon.connections)) - return (0, Pyro.constants.DENIED_SERVERTOOBUSY) - return (1,0) - def acceptIdentification(self, daemon, connection, token, challenge): - if "all" in self.allowedIDs: - return (1,0) - for authid in self.allowedIDs[:]: - if self.createAuthToken(authid, challenge, connection.addr, None, daemon) == token: - return (1,0) - Log.warn('DefaultConnValidator','connect authentication failed on conn ',connection) - return (0,Pyro.constants.DENIED_SECURITY) - def createAuthToken(self, authid, challenge, peeraddr, URI, daemon): - # Called from both client and server, is used to be able to validate the token. - # client: URI & peeraddr provided, daemon is None - # server: URI is None, peeraddr and daemon provided. - # Return hmac-md5 secure hash of our authentication phrase & the challenge. - return hmac.new(challenge, authid).digest() - def createAuthChallenge(self, tcpserver, conn): - # Server-side only, when new connection comes in. - # Challenge is secure hash of: server IP, process ID, timestamp, random value - # (NOTE: MUST RETURN EXACTLY AUTH_CHALLENGE_SIZE(=16) BYTES!) - try: - pid=os.getpid() - except: - pid=id(self) # at least jython has no getpid() - string = '%s-%d-%.20f-%.20f' %(str(getIPAddress()), pid, time.time(), random.random()) - return md5(string).digest() - def mungeIdent(self, ident): - # munge the identification string into something else that's - # not easily guessed or recognised, like the md5 hash: - return md5(ident).digest() - def setAllowedIdentifications(self, ids): - if ids is not None: - if type(ids) in (types.TupleType, types.ListType): - self.allowedIDs=map(self.mungeIdent, ids) # don't store ids themselves - else: - raise TypeError("ids must be a list") - else: - self.allowedIDs=["all"] # trick: allow all incoming authentications. - - -#-------- basic SSL connection validator, a specialized default validator. -class BasicSSLValidator(DefaultConnValidator): - def __init__(self): - DefaultConnValidator.__init__(self) - def acceptHost(self,daemon,connection): - (ok,code) = DefaultConnValidator.acceptHost(self, daemon, connection) - if ok: - peercert=connection.sock.get_peer_cert() - return self.checkCertificate(peercert) - return (ok,code) - def checkCertificate(self,cert): - # do something interesting with the cert here, in a subclass :) - if cert is None: - return (0,Pyro.constants.DENIED_SECURITY) - return (1,0) - - - -#-------- Helper class for local storage. -class LocalStorage(object): - def __init__(self): - self.caller=None - -#-------- TCPServer base class - - -class TCPServer(object): - def __init__(self, port, host='', threaded=_has_threading,prtcol='PYRO'): - self._ssl_server = 0 - self.connections = [] # connection threads - self.initTLS=lambda tls: None # default do-nothing func - if host: - HostUtil.get_inst().gethostbyname(host) # validate hostname - try: - if prtcol=='PYROSSL': - try: - from M2Crypto import SSL - except ImportError: - raise ProtocolError('SSL not available') - try: - self.ctx = SSL.Context('sslv23') - if Pyro.config.PYROSSL_KEY: - keyfile = os.path.join(Pyro.config.PYROSSL_CERTDIR, Pyro.config.PYROSSL_KEY) - else: - keyfile = None - self.ctx.load_cert(os.path.join(Pyro.config.PYROSSL_CERTDIR, Pyro.config.PYROSSL_CERT), - keyfile) - self.ctx.load_client_ca(os.path.join(Pyro.config.PYROSSL_CERTDIR, Pyro.config.PYROSSL_CA_CERT)) - self.ctx.load_verify_info(os.path.join(Pyro.config.PYROSSL_CERTDIR, Pyro.config.PYROSSL_CA_CERT)) - self.ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert,10) - self.ctx.set_allow_unknown_ca(1) - self._ssl_server = 1 - Log.msg('TCPServer','SSL Context initialized') - except: - Log.warn('TCPServer','SSL Context could not be initialized !!!') - self.setNewConnectionValidator(BasicSSLValidator()) - else: - self.setNewConnectionValidator(DefaultConnValidator()) - - # create server socket for new connections - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - set_reuse_addr(self.sock) - set_sock_no_inherit(self.sock) - self.sock.bind((host,port)) - self.sock.listen(Pyro.config.PYRO_TCP_LISTEN_BACKLOG) - if self._ssl_server: - self.sock = SSL.Connection(self.ctx,self.sock) # wrap server socket as SSL socket - # rest of members - self.threaded = threaded - self.mustShutdown=0 # global shutdown - self.localStorage=LocalStorage() # TLS for systems that don't have threads - return - except socket.error,msg: - raise ProtocolError(msg) - Log.msg('TCPServer','initialized') - - def __del__(self): - self.closedown(nolog=1) - - def setInitTLS(self, initTLS): - if not callable(initTLS): - raise TypeError("initTLS must be callable object") - self.initTLS=initTLS - # if in single thread mode, (re-)init the TLS right away. - if not Pyro.config.PYRO_MULTITHREADED: - self.initTLS(self.localStorage) - - def closedown(self, nolog=0): - # explicit closedown request - if len(self.connections)>0: - if not nolog: - Log.warn('TCPServer','Shutting down but there are still',len(self.connections),'active connections') - for c in self.connections[:]: - if isinstance(c,TCPConnection): - c.close() - if isinstance(c,Thread): - c.join() - self.connections=[] - if hasattr(self,'sock'): - self.sock.close() - del self.sock - - def setNewConnectionValidator(self,validator): - if not isinstance(validator, DefaultConnValidator): - raise TypeError("validator must be specialization of DefaultConnValidator") - self.newConnValidator=validator - def getNewConnectionValidator(self): - return self.newConnValidator - - def connectionHandler(self, conn): - # Handle the connection and all requests that arrive on it. - # This is only called in multithreading mode. - self.initTLS(self.getLocalStorage()) - try: - if self.getAdapter().handleConnection(conn, self): - Log.msg('TCPServer','new connection ',conn, ' #conns=',len(self.connections)) - while not self.mustShutdown: - try: - if not conn.connected: - # connection has been closed in the meantime! - raise ConnectionClosedError() - self.handleInvocation(conn) - except ConnectionClosedError: - # client went away. Exit immediately - self.removeConnection(conn) - return - except (PyroExceptionCapsule, Exception): - self.handleError(conn) - else: - # log entry has already been written by newConnValidator - self.removeConnection(conn) - finally: - # exiting thread. - self._removeFromConnectionList(None) - - def _removeFromConnectionList(self, obj): - if self.threaded and currentThread: - obj=currentThread() - try: - self.connections.remove(obj) - except ValueError: - pass - - - # this is the preferred way of dealing with the request loop. - def requestLoop(self, condition=lambda:1, timeout=3, others=[], callback=None): - while condition() and not self.mustShutdown: - self.handleRequests(timeout,others,callback) - - def handleRequests(self, timeout=None, others=[], callback=None): - if others and not callback: - raise ProtocolError('callback required') - if self.threaded: - self._handleRequest_Threaded(timeout,others,callback) - else: - self._handleRequest_NoThreads(timeout,others,callback) - - def _handleRequest_NoThreads(self,timeout,others,callback): - # self.connections is used to keep track of TCPConnections - socklist = self.connections+[self.sock]+others - ins,outs,exs = safe_select(socklist,[],[],timeout) - if self.sock in ins: - # it was the server socket, new incoming connection - ins.remove(self.sock) - if self._ssl_server: - try: - csock, addr = self.sock.accept() - #if not Pyro.config.PYROSSL_POSTCONNCHECK: - # csock.postConnectionCheck=None - except SSL.SSLError,error: - Log.warn('TCPServer','SSL error: '+str(error)) - return - else: - csock, addr = self.sock.accept() - - conn=TCPConnection(csock,addr) - if self.getAdapter().handleConnection(conn, self): - Log.msg('TCPServer','new connection ',conn, ' #conns=',len(self.connections)) - self.connections.append(conn) - else: - # connection denied, log entry has already been written by newConnValidator - self.removeConnection(conn) - - for c in ins[0:]: - if isinstance(c,TCPConnection): - ins.remove(c) - try: - self.handleInvocation(c) - if not c.connected: - self.removeConnection(c) - except ConnectionClosedError: - # client went away. - self.removeConnection(c) - except: - self.handleError(c) - - if ins and callback: - # the 'others' must have fired... - callback(ins) - - # def handleInvocation(self, conn): .... abstract method (implemented in subclass) - - - def _handleRequest_Threaded(self,timeout,others,callback): - # self.connections is used to keep track of connection Threads - socklist = [self.sock]+others - ins,outs,exs = safe_select(socklist,[],[],timeout) - if self.sock in ins: - # it was the server socket, new incoming connection - if self._ssl_server: - try: - csock, addr = self.sock.accept() - #if not Pyro.config.PYROSSL_POSTCONNCHECK: - # csock.postConnectionCheck=None - except SSL.SSLError,error: - Log.warn('TCPServer','SSL error: '+str(error)) - return - else: - csock, addr = self.sock.accept() - - conn=TCPConnection(csock,addr) - thread=Thread(target=self.connectionHandler, args=(conn,)) - thread.setDaemon(1) # thread must exit at program termination. - thread.localStorage=LocalStorage() - self.connections.append(thread) - thread.start() - elif callback: - # the 'others' must have fired... - callback(ins) - - def getLocalStorage(self): - # return storage object for this thread. - if self.threaded: - return currentThread().localStorage - else: - return self.localStorage - - # to be called if a dropped connection is detected: - def removeConnection(self, conn): - conn.close() - self._removeFromConnectionList(conn) - Log.msg('TCPServer','removed connection ',conn,' #conns=',len(self.connections)) - - # to be called to stop all connections and shut down. - def shutdown(self): - self.mustShutdown=1 - - def getAdapter(self): - raise NotImplementedError,'must be overridden to return protocol adapter' - def handleError(self,conn,onewaycall=False): - raise NotImplementedError,'must be overridden' - - def getServerSockets(self): - if self.threaded: - return [self.sock] - else: - return map(lambda conn: conn.sock, self.connections)+[self.sock] - -# Sometimes _selectfunction() raises an select.error exception with the EINTR -# errno flag set, which basically tells the caller to try again later. -# This safe_select method works around this case and indeed just tries again. -_selectfunction=select.select -if os.name=="java": - from select import cpython_compatible_select as _selectfunction -def safe_select(r,w,e,timeout=None): - delay=timeout - while True: - try: - # Make sure we don't delay longer than requested - start=time.time() - if delay is not None: - return _selectfunction(r,w,e,delay) - else: - return _selectfunction(r,w,e) - except select.error,x: - if x.args[0] == errno.EINTR or (hasattr(errno, 'WSAEINTR') and x.args[0] == errno.WSAEINTR): - delay=max(0.0,time.time()-start) - else: - raise diff --git a/lib/Pyro/test/__init__.py b/lib/Pyro/test/__init__.py deleted file mode 100644 index 6f66f15aed2..00000000000 --- a/lib/Pyro/test/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# just to make this a package. diff --git a/lib/Pyro/test/echoserver.py b/lib/Pyro/test/echoserver.py deleted file mode 100644 index dacce7c30aa..00000000000 --- a/lib/Pyro/test/echoserver.py +++ /dev/null @@ -1,101 +0,0 @@ -############################################################################# -# -# Pyro Echo Server, for test purposes -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -import sys -import time -from threading import Thread -import Pyro.core -import Pyro.naming -import Pyro.errors - -class EchoServer(Pyro.core.ObjBase): - verbose=False - def echo(self, args): - if self.verbose: - print ("%s - echo: %s" % (time.asctime(), args)) - return args - def error(self): - if self.verbose: - print ("%s - error: generating exception" % time.asctime()) - return 1//0 # division by zero error - - -class NameServer(Thread): - def __init__(self, hostname): - Thread.__init__(self) - self.setDaemon(1) - self.starter = Pyro.naming.NameServerStarter() - self.hostname=hostname - def run(self): - self.starter.start(hostname=self.hostname, dontlookupother=True) - def waitUntilStarted(self): - return self.starter.waitUntilStarted() - def getHostAndPort(self): - d=self.starter.daemon - return d.hostname, d.port - def shutdown(self): - self.starter.shutdown() - -def startNameServer(host): - ns=NameServer(host) - ns.start() - ns.waitUntilStarted() - return ns - -def main(args): - from optparse import OptionParser - parser=OptionParser() - parser.add_option("-H","--host", default="localhost", help="hostname to bind server on (default=localhost)") - parser.add_option("-p","--port", type="int", default=0, help="port to bind server on") - parser.add_option("-n","--naming", action="store_true", default=False, help="register with nameserver") - parser.add_option("-N","--nameserver", action="store_true", default=False, help="also start a nameserver") - parser.add_option("-v","--verbose", action="store_true", default=False, help="verbose output") - options,args = parser.parse_args(args) - - nameserver=None - if options.nameserver: - options.naming=True - nameserver=startNameServer(options.host) - print("") - - print ("Starting Pyro's built-in test echo server.") - d=Pyro.core.Daemon(host=options.host, port=options.port, norange=True) - echo=EchoServer() - echo.verbose=options.verbose - objectName=":Pyro.test.echoserver" - if options.naming: - host,port=None,None - if nameserver is not None: - host,port=nameserver.getHostAndPort() - ns=Pyro.naming.NameServerLocator().getNS(host,port) - try: - ns.createGroup(":Pyro.test") - except Pyro.errors.NamingError: - pass - d.useNameServer(ns) - if options.verbose: - print ("using name server at %s" % ns.URI) - else: - if options.verbose: - print ("not using a name server.") - uri=d.connect(echo, objectName) - print ("object name = %s" % objectName) - print ("echo uri = %s" % uri) - print ("echo uri = PYROLOC://%s:%d/%s" % (d.hostname, d.port, objectName)) - print ("echoserver running.") - try: - d.requestLoop() - finally: - d.shutdown(disconnect=True) - if nameserver is not None: - #nameserver.shutdown() - pass - -if __name__=="__main__": - main(sys.argv[1:]) diff --git a/lib/Pyro/util.py b/lib/Pyro/util.py deleted file mode 100644 index 96c196c6be4..00000000000 --- a/lib/Pyro/util.py +++ /dev/null @@ -1,610 +0,0 @@ -############################################################################# -# -# Pyro Utilities -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -from __future__ import with_statement -import os, sys, traceback -import time, random, linecache -import socket, binascii -import Pyro.constants -from Pyro.util2 import * # bring in 'missing' util functions - - -# bogus lock class, for systems that don't have threads. -class BogusLock(object): - def __enter__(self): - return self - def __exit__(self, exc_type, exc_val, exc_tb): - pass - def acquire(self): pass - def release(self): pass - -def getLockObject(): - if supports_multithreading(): # XXX - from threading import Lock - return Lock() - else: - return BogusLock() -def getRLockObject(): - if supports_multithreading(): - from threading import RLock - return RLock() - else: - return BogusLock() - - -# bogus event class, for systems that don't have threads -class BogusEvent(object): - def __init__(self): - self.flag=0 - def isSet(self): return self.flag==1 - def set(self): self.flag=1 - def clear(self): self.flag=0 - def wait(self,timeout=None): - raise RuntimeError("cannot wait in non-threaded environment") - - -class HostUtil(object): - """Wrap and memoize socket.gethost* results.""" - - EXPIRE = 3600.0 # singleton expires in 1 hour by default - _instance = None - - @classmethod - def get_inst(cls, new=False, expire=None): - """Return the singleton instance of this class. - - If "new" is True, create a new singleton instance. - - """ - if expire is None: - expire = cls.EXPIRE - if (cls._instance is None or new or - time.time() > cls._instance.expire_time): - cls._instance = cls(expire) - return cls._instance - - def __init__(self, expire): - self.expire_time = time.time() + expire - self._hostname = None - self._hostbyname = {} - self._hostbyaddr = {} - - def gethostbyaddr(self, addr): - """Wrap and memoize socket.gethostbyaddr.""" - if addr not in self._hostbyaddr: - self._hostbyaddr[addr] = socket.gethostbyaddr(addr) - return self._hostbyaddr[addr] - - def gethostbyname(self, name=None): - """Wrap and memoize socket.gethostbyname.""" - if name is None: - name = self.gethostname() - if name not in self._hostbyname: - self._hostbyname[name] = socket.gethostbyname(name) - return self._hostbyname[name] - - def gethostname(self): - """Wrap and memoize socket.gethostname.""" - if self._hostname is None: - self._hostname = socket.gethostname() - return self._hostname - - -def getEventObject(): - if supports_multithreading(): - from threading import Event - return Event() - else: - return BogusEvent() - - -# Logging stuff. - -# Select the logging implementation to use! -if Pyro.config.PYRO_STDLOGGING: - # new-style logging using logging module, python 2.3+ - import logging, logging.config - cfgfile=Pyro.config.PYRO_STDLOGGING_CFGFILE - if not os.path.isabs(cfgfile): - Pyro.config.PYRO_STDLOGGING_CFGFILE=os.path.join(Pyro.config.PYRO_STORAGE, cfgfile) - cfgfile=Pyro.config.PYRO_STDLOGGING_CFGFILE - externalConfig=0 - try: - open(cfgfile).close() - logging.config.fileConfig(cfgfile) - externalConfig=1 - except IOError,x: - # Config file couldn't be read! Use builtin config. - # First make the logfiles absolute paths: - if not os.path.isabs(Pyro.config.PYRO_LOGFILE): - Pyro.config.PYRO_LOGFILE=os.path.join(Pyro.config.PYRO_STORAGE, Pyro.config.PYRO_LOGFILE) - if not os.path.isabs(Pyro.config.PYRO_USER_LOGFILE): - Pyro.config.PYRO_USER_LOGFILE=os.path.join(Pyro.config.PYRO_STORAGE, Pyro.config.PYRO_USER_LOGFILE) - - class LoggerBase(object): - if externalConfig: - def __init__(self): - self.logger=logging.getLogger(self._getLoggerName()) - else: - def __init__(self): - self.logger=logging.getLogger("Pyro."+str(id(self))) # each time a different logger ... - self.setLevel(self._getPyroLevel()) - handler=logging.FileHandler(self._logfile()) - handler.setFormatter(logging.Formatter("%(asctime)s [%(process)d:%(thread)d] ** %(levelname)s ** %(message)s")) - self.logger.addHandler(handler) - def setLevel(self, pyroLevel): - if pyroLevel>=3: - self.logger.setLevel(logging.DEBUG) - elif pyroLevel>=2: - self.logger.setLevel(logging.WARN) - elif pyroLevel>=1: - self.logger.setLevel(logging.ERROR) - else: - self.logger.setLevel(999) - def msg(self,source,*args): - self.setLevel(self._getPyroLevel()) - if not args: - (args, source) = ([source], "N/A") - self.logger.info("%s ** %s", source, reduce(lambda x,y: str(x)+' '+str(y),args)) - def warn(self,source,*args): - self.setLevel(self._getPyroLevel()) - if not args: - (args, source) = ([source], "N/A") - self.logger.warn("%s ** %s", source, reduce(lambda x,y: str(x)+' '+str(y),args)) - def error(self,source,*args): - self.setLevel(self._getPyroLevel()) - if not args: - (args, source) = ([source], "N/A") - self.logger.error("%s ** %s", source, reduce(lambda x,y: str(x)+' '+str(y),args)) - def raw(self,ztr): - self.logger.log(999,ztr.rstrip()) - def _logfile(self): - raise NotImplementedError,'must override' - def _getlevel(self): - raise NotImplementedError,'must override' - - - class SystemLogger(LoggerBase): - def _getLoggerName(self): - return "Pyro.system" - def _getPyroLevel(self): - return Pyro.config.PYRO_TRACELEVEL - def _logfile(self): - return Pyro.config.PYRO_LOGFILE - - class UserLogger(LoggerBase): - def _getLoggerName(self): - return "Pyro.user" - def _getPyroLevel(self): - return Pyro.config.PYRO_USER_TRACELEVEL - def _logfile(self): - return Pyro.config.PYRO_USER_LOGFILE - -else: - # classic Pyro logging. - - class LoggerBase(object): - # Logger base class. Subclasses must override _logfile and _checkTraceLevel. - def __init__(self): - self.lock=getLockObject() - def msg(self,source,*args): - if self._checkTraceLevel(3): self._trace('NOTE',source, args) - def warn(self,source,*args): - if self._checkTraceLevel(2): self._trace('WARN',source, args) - def error(self,source,*args): - if self._checkTraceLevel(1): self._trace('ERR!',source, args) - def raw(self,str): - with self.lock: - f=open(self._logfile(),'a') - f.write(str) - f.close() - def _trace(self,typ,source, arglist): - with self.lock: - if not arglist: - (arglist, source) = ([source], "N/A") - try: - tf=open(self._logfile(),'a') - try: - pid=os.getpid() - pidinfo=" ["+str(os.getpid()) - except: - pidinfo=" [" # at least jython has no getpid() - if supports_multithreading(): - pidinfo+=":"+threading.currentThread().getName() - pidinfo+="] " - tf.write(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))+ - pidinfo+'** '+typ+' ** '+str(source)+' ** '+reduce(lambda x,y: str(x)+' '+str(y),arglist)+'\n') - tf.close() - except Exception,x: - pass - def _logfile(self): - raise NotImplementedError,'must override' - def _checkTraceLevel(self,level): - raise NotImplementedError,'must override' - - class SystemLogger(LoggerBase): - def _checkTraceLevel(self, level): - return Pyro.config.PYRO_TRACELEVEL >= level - def _logfile(self): - filename=Pyro.config.PYRO_LOGFILE - if not os.path.isabs(filename): - Pyro.config.PYRO_LOGFILE=os.path.join(Pyro.config.PYRO_STORAGE, filename) - return Pyro.config.PYRO_LOGFILE - - class UserLogger(LoggerBase): - def _checkTraceLevel(self, level): - return Pyro.config.PYRO_USER_TRACELEVEL >= level - def _logfile(self): - filename=Pyro.config.PYRO_USER_LOGFILE - if not os.path.isabs(filename): - Pyro.config.PYRO_USER_LOGFILE=os.path.join(Pyro.config.PYRO_STORAGE, filename) - return Pyro.config.PYRO_USER_LOGFILE - - -# The logger object 'Log'. -Log = SystemLogger() - - -# Caching directory lister, outputs (filelist,dirlist) tuple -# Based upon dircache.py, but implemented in a callable object -# that has a thread-safe cache. -class DirLister(object): - def __init__(self): - self.lock=getLockObject() - self.__listdir_cache = {} - - def __call__(self,path): - with self.lock: - try: - cached_mtime, files, directories = self.__listdir_cache[path] - del self.__listdir_cache[path] - except KeyError: - cached_mtime, files, directories = -1, [], [] - mtime = os.stat(path)[8] - if mtime <> cached_mtime: - files=[] - directories=[] - for e in os.listdir(path): - if os.path.isdir(os.path.join(path,e)): - directories.append(e) - else: - files.append(e) - with self.lock: - self.__listdir_cache[path] = mtime, files, directories - return files,directories - -listdir = DirLister() # callable object - - -# Fairly simple argument options parser. Like getopt(3). -class ArgParser(object): - def __init__(self): - pass - def parse(self, args, optionlist): - # optionlist is a string such as "ab:c" which means - # we search for 3 options (-a, -b, -c) of which -b has an argument. - self.options={} # public, the option->value dictionary - self.args=[] # public, the rest of the arguments - self.ignored=[] # public, ignored options - optionlist+=' ' # add sentinel - if type(args)==type(''): - args=args.split() - while args: - arg=args[0] - del args[0] - if arg[0]=='-': - if len(arg)>=2: # arg is an option. Check our list - idx = optionlist.find(arg[1]) - if idx>=0: - if optionlist[idx+1]==':': # option requires argument. - if len(arg)>=3: # argument is appended. Use this. - self.options[arg[1]]=arg[2:] - continue - # fetch argument from next string - if len(args)>=1: - self.options[arg[1]]=args[0] - del args[0] - continue - else: # missing arg, substitute None - self.options[arg[1]]=None - else: # option requires no argument, use None - self.options[arg[1]]=None - else: # didn't find this option, skip it - self.ignored.append(arg[1]) - else: # arg is a single '-'. Stop parsing. - for a in args: - self.args.append(a) - args=None - else: # arg is no option, add it to the residu list and continue - self.args.append(arg) - def hasOpt(self, option): - return self.options.has_key(option) - def getOpt(self, option, default=Exception()): - try: - return self.options[option] - except KeyError: - if not isinstance(default,Exception): - return default - raise KeyError('no such option') - def printIgnored(self): - if self.ignored: - print 'Ignored options:', - for o in self.ignored: - print '-'+o, - print - - -_getGUID_counter=0 # extra safeguard against double numbers -_getGUID_lock=getLockObject() - -if os.name=='java': - # define jython specific stuff - # first, the guid stuff. try java5 uuid first. - try: - from java.util import UUID - def getGUID(): - return str(UUID.randomUUID()) - except ImportError: - # older java, use rmi's vmid instead - from java.rmi.dgc import VMID - def getGUID(): - return str(VMID().toString().replace(':','-').replace('--','-')) - import imp - if not hasattr(imp,"acquire_lock"): - # simulate missing imp.acquire_lock() from jython 2.2 (fixed in jython 2.5) - imp_lock=getLockObject() - def imp_acquire_lock(): - return imp_lock.acquire() - def imp_release_lock(): - return imp_lock.release() - imp.acquire_lock=imp_acquire_lock - imp.release_lock=imp_release_lock - -elif sys.platform=='cli': - import System - def getGUID(): - # IronPython uses .NET guid call - return System.Guid.NewGuid().ToString() -else: - def getGUID(): - # Generate readable GUID string. - # The GUID is constructed as follows: hexlified string of - # AAAAAAAA-AAAABBBB-BBBBBBBB-BBCCCCCC (a 128-bit number in hex) - # where A=network address, B=timestamp, C=random. - # The 128 bit number is returned as a string of 16 8-bits characters. - # For A: should use the machine's MAC ethernet address, but there is no - # portable way to get it... use the IP address + 2 bytes process id. - try: - ip = HostUtil.get_inst().gethostbyname() - networkAddrStr=binascii.hexlify(socket.inet_aton(ip))+"%04x" % os.getpid() - except socket.error: - # can't get IP address... use another value, like our Python id() and PID - Log.warn('getGUID','Can\'t get IP address') - try: - ip=os.getpid() - except: - ip=0 - ip += id(getGUID) - networkAddrStr = "%08lx%04x" % (ip, os.getpid()) - - with _getGUID_lock: # cannot generate multiple GUIDs at once - global _getGUID_counter - t1=time.time()*100 +_getGUID_counter - _getGUID_counter+=1 - t2=int((t1*time.clock())%sys.maxint) & 0xffffff - t1=int(t1%sys.maxint) - timestamp = (long(t1) << 24) | t2 - r2=(random.randint(0,sys.maxint//2)>>4) & 0xffff - r3=(random.randint(0,sys.maxint//2)>>5) & 0xff - return networkAddrStr+'%014x%06x' % (timestamp, (r2<<8)|r3 ) - -def genguid_scripthelper(argv): - p=ArgParser() - p.parse(argv,'') - if p.args or p.ignored: - print 'Usage: genguid (no arguments)' - print 'This tool generates Pyro UIDs.' - raise SystemExit - print getGUID() - - - -# Get the configured pickling module. -# Currently supported: cPickle, pickle, gnosis.xml.pickle (@paranoia 0 or -1). -def getPickle(): - if Pyro.config.PYRO_XML_PICKLE: - # user requires xml pickle. Fails if that is not available! - return getXMLPickle() - else: - try: - import cPickle - return cPickle - except ImportError: - # Fall back on pickle if cPickle isn't available - import pickle - return pickle - -_xmlpickle={} -def getXMLPickle(impl=None): - # load & config the required xml pickle. - # Currently supported: Gnosis Utils' gnosis.xml.pickle. - global _xmlpickle - if not impl: - impl=Pyro.config.PYRO_XML_PICKLE - if impl in _xmlpickle: - return _xmlpickle[impl] - try: - if impl=='gnosis': - import gnosis.xml.pickle - import gnosis.version - gnosisVer=(gnosis.version.MAJOR, gnosis.version.MINOR) - if gnosisVer==(1,2): - # gnosis 1.2 style pickling, with paranoia setting - _xmlpickle[impl]=gnosis.xml.pickle - gnosis.xml.pickle.setParanoia(Pyro.config.PYRO_GNOSIS_PARANOIA) # default paranoia level is too strict for Pyro - gnosis.xml.pickle.setParser('SAX') # use fastest parser (cEXPAT?) - return gnosis.xml.pickle - elif gnosisVer>=(1,3): - from gnosis.xml.pickle import SEARCH_ALL, SEARCH_STORE, SEARCH_NO_IMPORT, SEARCH_NONE - if Pyro.config.PYRO_GNOSIS_PARANOIA<0: - class_search_flag = SEARCH_ALL # allow import of needed modules - elif Pyro.config.PYRO_GNOSIS_PARANOIA==0: - class_search_flag = SEARCH_NO_IMPORT # dont import new modules, only use known - else: - class_search_flag = SEARCH_STORE # only use class store - # create a wrapper class to be able to pass additional args into gnosis methods - class GnosisPickle: - def dumps(data, *args,**kwargs): - return gnosis.xml.pickle.dumps(data, allow_rawpickles=0) - dumps=staticmethod(dumps) - def loads(xml, *args, **kwargs): - return gnosis.xml.pickle.loads(xml, allow_rawpickles=0, class_search=class_search_flag) - loads=staticmethod(loads) - def dump(data, file, *args,**kwargs): - return gnosis.xml.pickle.dump(data, file, allow_rawpickles=0) - dump=staticmethod(dump) - def load(file, *args, **kwargs): - return gnosis.xml.pickle.load(file, allow_rawpickles=0, class_search=class_search_flag) - load=staticmethod(load) - _xmlpickle[impl]=GnosisPickle - return GnosisPickle - else: - raise NotImplementedError('no supported Gnosis tools version found (need at least 1.2). Found '+gnosis.version.VSTRING) - else: - raise ImportError('unsupported xml pickle implementation requested: %s' % impl) - except ImportError: - Log.error('xml pickling implementation (%s) is not available' % impl) - raise NotImplementedError('xml pickling implementation (%s) is not available' % impl) - - -# Pyro traceback printing -def getPyroTraceback(exc_obj, exc_type=None, exc_trb=None): - def formatRemoteTraceback(remote_tb_lines) : - result=[] - result.append(" +--- This exception occured remotely (Pyro) - Remote traceback:") - for line in remote_tb_lines : - if line.endswith("\n"): - line=line[:-1] - lines = line.split("\n") - for line in lines : - result.append("\n | ") - result.append(line) - result.append("\n +--- End of remote traceback\n") - return result - try: - if exc_type is None and exc_trb is None: - exc_type, exc_obj, exc_trb=sys.exc_info() - remote_tb=getattr(exc_obj,Pyro.constants.TRACEBACK_ATTRIBUTE,None) - local_tb=formatTraceback(exc_type, exc_obj, exc_trb) - if remote_tb: - remote_tb=formatRemoteTraceback(remote_tb) - return local_tb + remote_tb - else: - # hmm. no remote tb info, return just the local tb. - return local_tb - finally: - # clean up cycle to traceback, to allow proper GC - del exc_type, exc_obj, exc_trb - - -def formatTraceback(ex_type=None, ex_value=None, tb=None): - if ex_type is None and tb is None: - ex_type,ex_value,tb=sys.exc_info() - if Pyro.config.PYRO_DETAILED_TRACEBACK: - get_line_number = traceback.tb_lineno - - res = ['-'*50+ "\n", - " <%s> RAISED : %s\n" % (str(ex_type), str(ex_value)), - " Extended Stacktrace follows (most recent call last)\n", - '-'*50+'\n' ] - - try: - # Do some manipulation shit of stack - if tb != None: - frame_stack = [] - line_number_stack = [] - - #tb = sys.exc_info()[2] - while 1: - line_num = get_line_number(tb) - line_number_stack.append(line_num) - if not tb.tb_next: - break - tb = tb.tb_next - - f = tb.tb_frame - for x in line_number_stack: - frame_stack.append(f) - f = f.f_back - - frame_stack.reverse() - - lines = iter(line_number_stack) - seen_crap = 0 - for frame in frame_stack: - # Get items - flocals = frame.f_locals.items()[:] - - line_num = lines.next() - filename = frame.f_code.co_filename - - name = None - for key, value, in flocals: - if key == "self": - name = "%s::%s" % (value.__class__.__name__, frame.f_code.co_name) - if name == None: - name = frame.f_code.co_name - - res.append('File "%s", line (%s), in %s\n' % (filename, line_num, name)) - res.append("Source code:\n") - - code_line = linecache.getline(filename, line_num) - if code_line: - res.append(' %s\n' % code_line.strip()) - - if not seen_crap: - seen_crap = 1 - continue - - res.append("Local values:\n") - flocals.sort() - fcode=frame.f_code - for key, value, in flocals: - if key in fcode.co_names or key in fcode.co_varnames or key in fcode.co_cellvars: - local_res=" %20s = " % key - try: - local_res += repr(value) - except: - try: - local_res += str(value) - except: - local_res += "" - - res.append(local_res+"\n") - - res.append('-'*50 + '\n') - res.append(" <%s> RAISED : %s\n" % (str(ex_type), str(ex_value))) - res.append('-'*50+'\n') - return res - - except: - return ['-'*50+"\nError building extended traceback!!! :\n", - ''.join(traceback.format_exception(* sys.exc_info() ) ) + '-'*50 + '\n', - 'Original Exception follows:\n', - ''.join(traceback.format_exception(ex_type, ex_value, tb)) ] - - else: - # default traceback format. - return traceback.format_exception(ex_type, ex_value, tb) - - -def excepthook(ex_type, ex_value, ex_tb): - """An exception hook you can set sys.excepthook to, to automatically print remote Pyro tracebacks""" - traceback="".join(getPyroTraceback(ex_value,ex_type,ex_tb)) - sys.stderr.write(traceback) diff --git a/lib/Pyro/util2.py b/lib/Pyro/util2.py deleted file mode 100644 index 54b5fde1a42..00000000000 --- a/lib/Pyro/util2.py +++ /dev/null @@ -1,35 +0,0 @@ -############################################################################# -# -# Pyro Utilities (part 2, to avoid circular dependencies) -# User code should never import this, always use Pyro.util! -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -_supports_mt=None -_supports_comp=None - -def supports_multithreading(): - global _supports_mt - if _supports_mt is None: - try: - from threading import Thread, Lock - _supports_mt=1 - except: - _supports_mt=0 - return _supports_mt - -def supports_compression(): - global _supports_comp - if _supports_comp is None: - try: - import zlib - _supports_comp=1 - except: - _supports_comp=0 - return _supports_comp - -if supports_multithreading(): - import threading diff --git a/lib/Pyro/wxnsc.py b/lib/Pyro/wxnsc.py deleted file mode 100644 index 6012140e67d..00000000000 --- a/lib/Pyro/wxnsc.py +++ /dev/null @@ -1,914 +0,0 @@ -#!/usr/bin/env python - -""" -A wxPython gui to nsc (Pyro Name Server Control tool). -This gui doesn't have as many features as the xnsc that ships with Pyro, -but it has some nice features that the xnsc doesn't have ;) - -'Pyro' - Python Remote Objects is -(c) Irmen de Jong - irmen@razorvine.net - -This file 'wxnsc.py' is -(c) Jan Finell - finell@users.sourceforge.net - - -Usage (from the commandline): - -# to use set PYRO environment variables or broadcasting -# for finding the nameserver host... -> wxnsc.py - -""" - -__author__ = "Jan Finell" -__date__ = "$Date: 2009/03/27 14:30:29 $" -__revision__ = "$Revision: 1.10.2.6 $" - -# -# Standard modules -# -import os, sys -import traceback, cStringIO - -# -# GUI modules -# -import wx - -# -# Pyro modules -# -from Pyro.naming import NameServerLocator -from Pyro.errors import NamingError, ConnectionClosedError,\ - ConnectionDeniedError -import Pyro.core -from Pyro.util import HostUtil - -#----------------------------------------------------------------------# -# Module constants -DEFAULT_GROUPNAME = ':Default' -PROTECTED_GROUPS = [DEFAULT_GROUPNAME, ':Pyro'] - -GROUP_XPM = [ -"16 16 9 1", -" c None", -". c #FFFFFF", -"+ c #000000", -"@ c #C3C3C3", -"# c #A0A0A0", -"$ c #8080FF", -"% c #585858", -"& c #FFFFFF", -"* c #808080", -" +%+ ", -" +%$$++ ", -" +&%%$$+++++ ", -" +&&@%%$$$$$+ ", -" +&@&@@%$$$$% ", -" +@&@@@@%%%%$+ ", -" +&@@@@@@@#@+$% ", -" +@@@@@@@#@#+$% ", -" +@@@#@@#@##+$% ", -" +@@@@@#@###+$% ", -" ++*@@#@####+$% ", -" ++*@#####+$% ", -" ++#####+$% ", -" ++###+$++", -" ++#+$++", -" ++++ "] - -GROUP_OPEN_XPM =[ -"16 16 12 1", -" c None", -". c #FFFFFF", -"+ c #000000", -"@ c #8080FF", -"# c #808080", -"$ c #C3C3C3", -"% c #C0C0FF", -"& c #A0A0A0", -"* c #303030", -"= c #FFFFFF", -"- c #DCDCDC", -"; c #585858", -" +++ ", -" +@#++ ", -" +@%@#++++ ", -"+++ +@%%%%%%@+ ", -"+&&*;@@@%%%%%@+ ", -"+#&&&#@@@@%%%@#+", -" *&&&&##@@@@%@#+", -" +#&&$$&##@@@@#+", -" *&$&$$$$##@@#+", -" +#$$$$-$-&@@#+", -" *#-$$$-=%;@#+", -" +*&--===&@#+", -" +*$===%;#+", -" +*$==##+", -" +*$=#+", -" +**+"] - -ITEM_XPM = [ -"16 16 11 1", -" c None", -". c #FFFFFF", -"+ c #FFFFFF", -"@ c #000000", -"# c #585858", -"$ c #DCDCDC", -"% c #FFFFC0", -"& c #FFDCA8", -"* c #303030", -"= c #C3C3C3", -"- c #A0A0A0", -" #####**@@ ", -" #$$$$$=*=@ ", -" #$+++++#+$@ ", -" #$+++++#$+=@", -" #$+++++###*@", -" #$++++++$=-@", -" #$++++++%+$@", -" #$+++++++++@", -" #$++++%+%+%@", -" #$+++++++&+@", -" #$++%+%+%+%@", -" #$+++++&+&+@", -" #$%+%+%+%+%@", -" #$+++&+&+&%@", -" #$%+%+%+%%%@", -" *@@@@@@@@@@@"] -#----------------------------------------------------------------------# -# Helper functions - -def cmp_name(n1, n2): - return cmp(n1[0], n2[0]) - -def show_message_dialog(parent, msg, title, style): - if sys.platform[:3] == 'win': - dlg = WinMessageDialog(parent, msg, title, style) - else: - dlg = wx.MessageDialog(parent, msg, title, style) - dlg.CentreOnParent() - retval = dlg.ShowModal() - dlg.Destroy() - return retval - -#----------------------------------------------------------------------# -## Classes - - -class wx_StdoutLog(wx.TextCtrl): - """ - :Purpose: A simple text ctrl that can be used for logging standard out - """ - def write(self, data): - if data.strip(): - wx.TextCtrl.AppendText(self, '%s\n' % data) - -class wx_NSC(wx.Frame): - """ - :Purpose: The main frame of the GUI. - """ - def __init__(self, nsHost, nsPort, bcAddr): - """ - :Parameters: - - `nsHost`: the name server host to connect to. This is the - name of the host or the ip. - - `nsPort`: the name server port. By default the Pyro name - server port is 9090 - - `bcAddr`: override for the broadcast address. - """ - wx.Frame.__init__(self, None, -1, 'Pyro Name Server') - self.nsHost = nsHost - self.nsPort = nsPort - self.bcAddr = bcAddr - self.NS = None - - self._build() - imageList = wx.ImageList(16,16) - self.__idGroup = imageList.Add(wx.BitmapFromXPMData(GROUP_XPM)) - self.__idItem = imageList.Add(wx.BitmapFromXPMData(ITEM_XPM)) - self.__idGroupOpen = imageList.Add(wx.BitmapFromXPMData(GROUP_OPEN_XPM)) - self.treeCtrlItems.SetImageList(imageList) - self.__imageList = imageList - - self._bindEvents() - # binding stdout to my own txtCtrl Log. - sys.stdout = self.txtCtrlLog - - self._log("Pyro version: "+Pyro.constants.VERSION) - - self.nsc_findNS() - if self.NS: self.update() - - #-- public methods --# - def enable(self, enable=True): - """ - Enabling/disabling some of the buttons. - """ - self.buttonDeleteGroup.Enable(enable) - self.buttonCreateGroup.Enable(enable) - self.buttonDeleteSelected.Enable(enable) - self.buttonRegisterItem.Enable(enable) - self.buttonShowMeta.Enable(enable) - self.buttonSetMeta.Enable(enable) - - def update(self): - """ - """ - tree = self.treeCtrlItems - tree.DeleteAllItems() - root = tree.AddRoot(':') - tree.SetItemImage(root, self.__idGroup) - tree.SetItemImage(root, self.__idGroupOpen, - wx.TreeItemIcon_Expanded) - self._populate_tree(tree, root, ':') - - # enabling/disabling buttons, depending on the current state. - self.enable(self.NS != None) - - def _populate_tree(self, tree, parent, group): - subgroupsL = self.nsc_list_groups(group) - subgroupsL.sort() - itemsL = self.nsc_list_items(group) - itemsL.sort() - for subgroup in subgroupsL: - groupB = tree.AppendItem(parent, subgroup) - tree.SetPyData(groupB, 0) - tree.SetItemImage(groupB, self.__idGroup) - tree.SetItemImage(groupB, self.__idGroupOpen, - wx.TreeItemIcon_Expanded) - self._populate_tree(tree, groupB, subgroup) - for item in itemsL: - itemB = tree.AppendItem(parent, item) - tree.SetPyData(itemB, 1) - tree.SetItemImage(itemB, self.__idItem) - tree.SetItemImage(itemB, self.__idItem, - wx.TreeItemIcon_Selected) - #-- nsc methods --# - def nsc_findNS(self, ident=None): - """ - Locating the Name Server by using given nsHost and nsPort - """ - locator = NameServerLocator(identification=ident) - try: - if self.nsHost: - self._log('connecting to Name Server (%s:%s)' % (self.nsHost, - self.nsPort)) - self.NS = locator.getNS(self.nsHost, self.nsPort, trace=1, bcaddr=self.bcAddr) - else: - self._log('broadcasting to find Name Server') - self.NS = locator.getNS(None, None, trace = 1, bcaddr=self.bcAddr) - self.nsHost = self.NS.URI.address - self.nsPort = self.NS.URI.port - self.NS._setIdentification(ident) - self._log('Name Server found, URI = %s' % self.NS.URI) - self._setNSData() - except ConnectionDeniedError, e: - if str(e).find( Pyro.constants.deniedReasons[Pyro.constants.DENIED_SECURITY] ) != -1: - msg = 'Authentication required:' - dlg = wx.TextEntryDialog(self, msg, 'Authentication', - style=wx.OK|wx.CANCEL|wx.TE_PASSWORD) - dlg.CentreOnParent() - if dlg.ShowModal() == wx.ID_OK: - ident = dlg.GetValue() - self.nsc_findNS(ident) - else: - self.NS = None - self._log('Connection to Name Server denied!','error') - else: - self.NS = None - self._logError('Unable to connect to Name Server') - except: - self.NS = None - self._logError('Name Server not found!') - - - def nsc_list_groups(self, ingroup): - """ - Returns a list of group names inside given group. - """ - return self._nsc_list(ingroup, 0) - - def nsc_list_items(self, ingroup): - """ - Returns a list of item names inside given group. - """ - return self._nsc_list(ingroup, 1) - - def _nsc_list(self, ingroup, type): - """ - Generic method for listing either groups or items inside a given group. - - type = 0 : group - type = 1 : item - """ - items = [] - if self.NS: - for name, t in self.NS.list(ingroup): - if t == type: - if type == 1: - uri = self.NS.resolve('%s.%s' % (ingroup,name)) - name = '%s (%s)' % (name, uri) - elif ingroup != ':': - name = '%s.%s' % (ingroup, name) - else: - name = '%s%s' % (ingroup, name) - items.append(name) - return items - - def nsc_create_group(self, groupName): - """ - Creating given group - """ - if self.NS: - try: - self.NS.createGroup(groupName) - self._log('created group (%s)' % (groupName)) - return 1 - except NamingError, e: - self._logError('unable to create group %s because %s' % (groupName, - e)) - return 0 - - def nsc_delete_group(self, groupName): - """ - Deleting given group - """ - if self.NS: - try: - self.NS.deleteGroup(groupName) - self._log('group %s deleted' % groupName) - return 1 - except NamingError, e: - self._logError('unable to delete group %s because %s' % (groupName, - e)) - return 0 - - def nsc_ping(self): - """ - Ping the current Name Server - """ - if self.NS: - try: - self.NS.ping() - self._log('ping Name Server (%s): up and running' % self.nsHost) - except: - self._logError('Name Server not responding.') - else: - self._logError('Name Server not responding') - - def nsc_delete(self, name): - """Removing given name from the Name Server. - - :Parameters: - - `name`: the name to delete from the Name Server - """ - try: - self.NS.unregister(name) - self._log('%s deleted successfully' % name) - return 1 - except NamingError, e: - self._logError('unable to delete %s because %s' % (name, e)) - except: - self._logError('deletion of %s failed' % name) - return 0 - - def nsc_register_item(self, name, uri): - """ - Registering new item with given name and uri - """ - try: - self.NS.register(name, uri) - uri = Pyro.core.PyroURI(uri) - self._log('registered %s with %s' % (name, uri)) - return 1 - except NamingError, e: - self._logError('unable to register,\nName Server error: %s' % e) - except Exception, e: - self._logError('unable to register, error: %s' % e) - return 0 - - def nsc_set_meta(self, name, meta): - """ - Set user meta data - """ - try: - self.NS.setMeta(name, meta) - self._log('set user meta data on '+name) - return 1 - except NamingError, e: - self._logError('unable to set user meta data,\nName Server error: %s' % e) - except Exception, e: - self._logError('unable to set user meta data, error: %s' % e) - return 0 - - def nsc_show_meta(self, name): - fullName = self.NS.fullName(name) - try: - self._log('"%s" system meta info: %s' % (fullName, - self.NS._getSystemMeta(fullName))) - self._log('"%s" user meta info: %s' % (fullName, - self.NS.getMeta(name))) - except NamingError, e: - self._logError('unable to get meta info,\nName Server error: %s' % e) - except Exception, e: - self._logError('unable to get meta info, error: %s' % e) - - #-- gui event methods --# - def OnCheckNS(self, event): - if self._checkNS(): - self.update() - - def OnClose(self, event): - sys.stdout = sys.__stdout__ #restoring the stdout - self.Destroy() - - def OnCreateGroup(self, event): - """ - Creating group in selected parent - """ - tree = self.treeCtrlItems - items = tree.GetSelections() - if items: - if tree.GetPyData(items[0]) == 0: - # use the selected group - parentGroupI = items[0] - parentGroupName = tree.GetItemText(parentGroupI) - else: - # take the parent - parentGroupI = tree.GetItemParent(items[0]) - parentGroupName = tree.GetItemText(parentGroupI) - else: - parentGroupI = tree.GetRootItem() - parentGroupName = ':' - msg = 'Create group in "%s", with name:' % parentGroupName - dlg = wx.TextEntryDialog(self, msg, 'Enter group name') - dlg.CentreOnParent() - if dlg.ShowModal() == wx.ID_OK: - if parentGroupName != ':': - groupName = '%s.%s' % (parentGroupName, dlg.GetValue()) - else: - groupName = ':%s' % (dlg.GetValue()) - if self.nsc_create_group(groupName): - groupI = tree.AppendItem(parentGroupI, groupName) - tree.SetPyData(groupI, 0) - tree.SetItemImage(groupI, self.__idGroup) - tree.SetItemImage(groupI, self.__idGroupOpen, - wx.TreeItemIcon_Expanded) - tree.Expand(parentGroupI) - - def OnSetMeta(self, ev=None): - """ - set user meta on selected groups + items - """ - tree = self.treeCtrlItems - itemsL, groupsL = self._getSelections() - namesL = itemsL + groupsL - if namesL: - namesS = ',\n '.join(namesL) - msg = 'User meta data string for:\n %s' % namesS - dlg = wx.TextEntryDialog(self, msg, 'Enter meta data') - dlg.CentreOnParent() - if dlg.ShowModal() == wx.ID_OK: - meta=dlg.GetValue() - for name in namesL: - self.nsc_set_meta(name,meta) - - def OnDelete(self, ev=None): - """ - Deleting selected items. - """ - tree = self.treeCtrlItems - itemsL = tree.GetSelections() - namesL = [] - deleteL = [] - for i in itemsL: - # only items (ie. no groups) - if tree.GetPyData(i) == 1: - parent = tree.GetItemParent(i) - parentName = tree.GetItemText(parent) - name = tree.GetItemText(i).split()[0] #only name - namesL.append('%s.%s' % (parentName, name)) - deleteL.append(i) - if namesL: - namesS = ',\n '.join(namesL) - ret = show_message_dialog(self, - 'Really delete following name(s)?:\n %s' % namesS, - '-- Confirm --', wx.YES|wx.NO|wx.ICON_QUESTION) - if ret == wx.ID_YES: - for name, i in zip(namesL,deleteL): - if self.nsc_delete(name): - tree.Delete(i) - - def OnDeleteGroup(self, ev=None): - """ - Deleting selected groups. - """ - tree = self.treeCtrlItems - itemsL = tree.GetSelections() - namesL = [] - deleteL = [] - for i in itemsL: - # only groups (ie. no items) - if tree.GetPyData(i) == 0: - name = tree.GetItemText(i) - if name not in PROTECTED_GROUPS and tree.GetChildrenCount(i)==0: - namesL.append(name) - deleteL.append(i) - if namesL: - namesS = ',\n'.join(namesL) - ret = show_message_dialog(self, - 'Really delete following group(s)?:\n %s' % namesS, - '-- Confirm --', wx.YES|wx.NO|wx.ICON_QUESTION) - if ret == wx.ID_YES: - for name, i in zip(namesL, deleteL): - if self.nsc_delete_group(name): - tree.Delete(i) - - def OnKeyPressed(self, event): - """ - Calling delete for both items and groups - """ - if event.GetKeyCode() == 127: - # deleting both selected groups and items - self.OnDelete() - self.OnDeleteGroup() - if event.GetKeyCode() == 105: # 105 == 'i' - # showing meta information on selected item - self.OnShowMeta() - event.Skip() - - def OnPing(self, event): - if self._checkNS(): - self.update() - self.nsc_ping() - - - def OnRegisterItem(self, event): - """ - Registering item in selected parent. - """ - tree = self.treeCtrlItems - items = tree.GetSelections() - if items: - if tree.GetPyData(items[0]) == 0: - # use the selected group - parentGroupI = items[0] - parentGroupName = tree.GetItemText(parentGroupI) - else: - parentGroupI = tree.GetItemParent(items[0]) - parentGroupName = tree.GetItemText(parentGroupI) - else: - parentGroupI = tree.GetRootItem() - parentGroupName = ':' - - msg = 'Register new item in "%s", with:\n ' % parentGroupName - dlg = wx.TextEntryDialog(self, msg, 'Register item') - dlg.CentreOnParent() - if dlg.ShowModal() == wx.ID_OK: - try: - itemName, uri = dlg.GetValue().split() - except: - self._log('Invalid arguments, use ', 'error') - else: - if parentGroupName != ':': - itemName = '%s.%s' % (parentGroupName, itemName) - else: - itemName = ':%s' % (itemName) - if self.nsc_register_item(itemName, uri): - label = '%s (%s)' % (dlg.GetValue().split()[0], uri) - itemI = tree.AppendItem(parentGroupI, label) - tree.SetPyData(itemI, 1) - tree.SetItemImage(itemI, self.__idItem) - tree.SetItemImage(itemI, self.__idItem, - wx.TreeItemIcon_Selected) - tree.Expand(parentGroupI) - - def OnUpdate(self, event): - self._checkNS() - self.update() - - def OnShowMeta(self, event=None): - itemsL, groupsL = self._getSelections() - for name in itemsL + groupsL: - self.nsc_show_meta(name) - - - #-- protected methods --# - def _checkNS(self): - """ - Reads the new values from the txtCtrlNSHost and txtCtrlNSPort. - If changed, it tries to connect to the new Name Server. - """ - changed = 0 - if self.txtCtrlNSHost.IsModified(): - self.nsHost = self.txtCtrlNSHost.GetValue() - changed = 1 - if self.txtCtrlNSPort.IsModified(): - try: - port = int(self.txtCtrlNSPort.GetValue()) - self.nsPort = port - except ValueError: - self._logError('Integer required for port') - changed = 1 - if changed: - self.nsc_findNS() - return changed - - def _log(self, line, status='info'): - """Writing given line to the log-textCtrl. - - :Parameters: - - `line`: text to log - - `status`: status should be 'info' or 'error'. If 'info' the - text will be colored blue, if 'error' the text will - be red. - """ - start = self.txtCtrlLog.GetLastPosition() - self.txtCtrlLog.AppendText('%s\n' % line) - color = wx.BLACK - if status == 'info': - color = wx.BLUE - elif status == 'error': - color = wx.RED - self.txtCtrlLog.SetStyle(start, self.txtCtrlLog.GetLastPosition(), - wx.TextAttr(color)) - - def _logError(self, line): - """ - Getting the traceback of previous error, and logging this. - """ - a, b, tb = sys.exc_info() - if a == ConnectionClosedError: - self.NS = None - self._log('Connection with Name Server lost', 'error') - self.enable(False) - buf = cStringIO.StringIO() - traceback.print_exc(file = buf) - self._log('%s:\n%s' % (line, buf.getvalue()), 'error') - - def _setNSData(self): - """ - Updates the display of current Name Server information. - """ - try: - ns_name, t, ns_ip = HostUtil.get_inst().gethostbyaddr(self.nsHost) - ns_ip = ns_ip[0] - except: - ns_name, ns_ip = self.nsHost, '' - self.txtCtrlNSHost.SetValue('%s' % ns_name) - self.txtCtrlNSPort.SetValue('%s' % self.nsPort) - self.SetTitle('Pyro Name Server ( %s - %s )' % (ns_name, ns_ip)) - - def _getSelections(self): - tree = self.treeCtrlItems - selectionsL = tree.GetSelections() - itemsL = [] - groupsL = [] - for i in selectionsL: - if tree.GetPyData(i) == 0: - # group - groupsL.append(tree.GetItemText(i)) - elif tree.GetPyData(i) == 1: - # item - parentName = tree.GetItemText(tree.GetItemParent(i)) - name = tree.GetItemText(i).split()[0] - itemsL.append('%s.%s' % (parentName, name)) - return itemsL, groupsL - - #-- build / bind methods --# - def _bindEvents(self): - """ - Binding events to the gui widgets. - """ - wx.EVT_BUTTON(self, self.buttonPing.GetId(), self.OnPing) - wx.EVT_BUTTON(self, self.buttonUpdate.GetId(), self.OnUpdate) - wx.EVT_BUTTON(self, self.buttonClose.GetId(), self.OnClose) - wx.EVT_BUTTON(self, self.buttonDeleteGroup.GetId(), self.OnDeleteGroup) - wx.EVT_BUTTON(self, self.buttonCreateGroup.GetId(), self.OnCreateGroup) - wx.EVT_BUTTON(self, self.buttonDeleteSelected.GetId(), self.OnDelete) - wx.EVT_BUTTON(self, self.buttonRegisterItem.GetId(), self.OnRegisterItem) - wx.EVT_BUTTON(self, self.buttonShowMeta.GetId(), self.OnShowMeta) - wx.EVT_BUTTON(self, self.buttonSetMeta.GetId(), self.OnSetMeta) - wx.EVT_TEXT_ENTER(self, self.txtCtrlNSHost.GetId(), self.OnCheckNS) - wx.EVT_TEXT_ENTER(self, self.txtCtrlNSPort.GetId(), self.OnCheckNS) - wx.EVT_CHAR(self.treeCtrlItems, self.OnKeyPressed) - - def _build(self): - """ - Building widgets and setting static widget data. - """ - parent = wx.Panel(self, -1) - sizer0 = wx.BoxSizer(wx.VERTICAL) - sizer0.Add(self._buildTopBar(parent), 0, wx.ALIGN_LEFT|wx.GROW, 5) - - splitter = wx.SplitterWindow(parent, -1) - #- TOP PART --------------------------------------------------------# - topParent = wx.Panel(splitter, -1) - topSizer = wx.BoxSizer(wx.VERTICAL) - self.treeCtrlItems = wx.TreeCtrl(topParent, -1, - style = wx.TR_TWIST_BUTTONS|wx.TR_LINES_AT_ROOT|wx.TR_HAS_BUTTONS|wx.TR_HIDE_ROOT|wx.TR_MULTIPLE) - topSizer.Add(self.treeCtrlItems, 1, wx.EXPAND, 5) - topParent.SetAutoLayout( True ) - topParent.SetSizer(topSizer ) - topSizer.Fit(topParent) - topSizer.SetSizeHints(topParent) - #-------------------------------------------------------------------# - #- BOTTOM PART -----------------------------------------------------# - bottomParent = wx.Panel(splitter,-1) - bottomSizer = wx.BoxSizer(wx.VERTICAL) - self.txtCtrlLog=wx_StdoutLog(bottomParent, -1, "", - size= wx.Size(-1, 10), - style=wx.TE_MULTILINE|wx.TE_READONLY|wx.TE_RICH) - bottomSizer.Add(self.txtCtrlLog, 1, wx.EXPAND, 5) - bottomParent.SetAutoLayout( True ) - bottomParent.SetSizer(bottomSizer ) - bottomSizer.Fit(bottomParent) - bottomSizer.SetSizeHints(bottomParent) - #-------------------------------------------------------------------# - splitter.SplitHorizontally(topParent,bottomParent, -100) - sizer0.Add(splitter, 1, wx.EXPAND|wx.ALIGN_CENTRE, 5) - - self.buttonClose = wx.Button(parent, -1, 'Close') # buttonClose - sizer0.Add(self.buttonClose, 0, wx.ALIGN_CENTRE|wx.ALL, 5) - - parent.SetAutoLayout( True ) - parent.SetSizer( sizer0 ) - sizer0.Fit( parent) - sizer0.SetSizeHints( parent) - - def _buildTopBar(self, parent): - """ - Widget building - """ - sizer0 = wx.BoxSizer(wx.VERTICAL) - #-- - sizer1 = wx.BoxSizer(wx.HORIZONTAL) - txt1 = wx.StaticText(parent, -1, 'Name Server:') - txt1.SetForegroundColour(wx.BLUE) - sizer1.Add(txt1, 0, wx.ALIGN_LEFT|wx.ALIGN_CENTRE|wx.ALL, 5) - self.txtCtrlNSHost = wx.TextCtrl(parent, -1, '', size=wx.Size(300,-1), - style=wx.TE_PROCESS_ENTER) - sizer1.Add(self.txtCtrlNSHost, 0, - wx.ALIGN_LEFT|wx.ALIGN_BOTTOM|wx.TOP|wx.BOTTOM, 5) - txtColon = wx.StaticText(parent, -1, ':') - txtColon.SetForegroundColour(wx.BLUE) - sizer1.Add(txtColon, 0, - wx.ALIGN_LEFT|wx.ALIGN_CENTRE|wx.TOP|wx.BOTTOM, 5) - self.txtCtrlNSPort = wx.TextCtrl(parent, -1, '', size=wx.Size(50,-1), - style=wx.TE_PROCESS_ENTER) - sizer1.Add(self.txtCtrlNSPort, 0, - wx.ALIGN_LEFT|wx.ALIGN_BOTTOM|wx.TOP|wx.BOTTOM, 5) - self.buttonUpdate = wx.Button(parent, -1, 'Update') # buttonUpdate - sizer1.Add(self.buttonUpdate, 0, wx.ALIGN_LEFT|wx.ALL, 5) - self.buttonPing = wx.Button(parent, -1, 'Ping') # buttonPing - sizer1.Add(self.buttonPing, 0, wx.ALIGN_LEFT|wx.ALL, 5) - sizer0.Add(sizer1, 0, wx.ALIGN_LEFT|wx.GROW, 5) - #-- - lineH1 = wx.StaticLine(parent, -1, style=wx.LI_HORIZONTAL) - sizer0.Add(lineH1, 0, wx.GROW|wx.ALIGN_CENTRE|wx.LEFT|wx.RIGHT, 5) - #-- - sizer2 = wx.BoxSizer(wx.HORIZONTAL) - self.buttonDeleteGroup = wx.Button(parent, -1, ' Delete group(s) ') - sizer2.Add(self.buttonDeleteGroup, 0, - wx.ALIGN_LEFT|wx.ALIGN_CENTER|wx.TOP|wx.LEFT|wx.BOTTOM, 5) - self.buttonCreateGroup = wx.Button(parent, -1, ' Create group... ') - sizer2.Add(self.buttonCreateGroup, 0, - wx.ALIGN_LEFT|wx.ALIGN_CENTER|wx.TOP|wx.LEFT|wx.BOTTOM, 5) - lineV1 = wx.StaticLine(parent, -1, style=wx.LI_VERTICAL) - sizer2.Add(lineV1, 0, - wx.ALL|wx.GROW, 5) - self.buttonDeleteSelected = wx.Button(parent, -1, - ' Delete item(s) ') - sizer2.Add(self.buttonDeleteSelected, 0, - wx.ALIGN_LEFT|wx.ALIGN_CENTER|wx.ALL, 5) - self.buttonRegisterItem = wx.Button(parent, -1, ' Register item... ') - sizer2.Add(self.buttonRegisterItem, 0, - wx.ALIGN_LEFT|wx.ALIGN_CENTER|wx.TOP|wx.BOTTOM, 5) - lineV2 = wx.StaticLine(parent, -1, style=wx.LI_VERTICAL) - sizer2.Add(lineV2, 0, - wx.ALL|wx.GROW, 5) - self.buttonShowMeta = wx.Button(parent, -1, ' Show meta ') - sizer2.Add(self.buttonShowMeta, 0, - wx.ALIGN_LEFT|wx.ALIGN_CENTER|wx.TOP|wx.BOTTOM, 5) - self.buttonSetMeta = wx.Button(parent, -1, ' Set meta ') - sizer2.Add(self.buttonSetMeta, 0, - wx.ALIGN_LEFT|wx.ALIGN_CENTER|wx.TOP|wx.BOTTOM, 5) - sizer0.Add(sizer2, 0, wx.ALIGN_LEFT, 5) - #-- - return sizer0 - -#----------------------------------------------------------------------# -class WinMessageDialog(wx.Dialog): - ''' - :Purpose: Message dialog for MS Win. - The parameters are the same as for wx.MessageDialog - - :Detail: On Windows the native wx.MessageDialog can not - be centered on top of the parent or positioned, ie. - it will always be centered on the screen. - - ''' - def __init__(self, parent=None, message='Message:', - caption='Message', style=wx.OK|wx.CANCEL,pos=wx.DefaultPosition): - wx.Dialog.__init__(self, parent, -1, caption, size=wx.DefaultSize, - style=wx.CAPTION, pos=pos) - self._build(message, style) - self.Fit() - - def OnButton(self, ev): - self.EndModal(ev.GetId()) - - def _build(self, msg, style): - parent = wx.Panel(self, -1) - sizer = wx.BoxSizer(wx.VERTICAL) - #-- icon and message --# - msgSizer = wx.BoxSizer(wx.HORIZONTAL) - - # icon # - artID = None - if style & wx.ICON_EXCLAMATION == wx.ICON_EXCLAMATION \ - or style & wx.ICON_HAND == wx.ICON_HAND: - artID = wx.ART_WARNING - elif style & wx.ICON_ERROR == wx.ICON_ERROR: - artID = wx.ART_ERROR - elif style & wx.ICON_QUESTION == wx.ICON_QUESTION: - artID = wx.ART_QUESTION - elif style & wx.ICON_INFORMATION == wx.ICON_INFORMATION: - artID = wx.ART_INFORMATION - if artID: - bmp = wx.ArtProvider_GetBitmap(artID, wx.ART_MESSAGE_BOX, (48,48)) - bmpIcon = wx.StaticBitmap(parent, -1, bmp) - msgSizer.Add(bmpIcon, 0, wx.ALIGN_CENTRE|wx.ALL, 5) - - # msg # - txtMsg = wx.StaticText(parent, -1, msg, style=wx.ALIGN_CENTRE) - msgSizer.Add(txtMsg, 0, wx.ALIGN_CENTRE|wx.ALL, 5) - - sizer.Add(msgSizer, 0, wx.ALIGN_CENTRE, 5) - line = wx.StaticLine(parent, -1, style=wx.LI_HORIZONTAL) - sizer.Add(line, 0, wx.GROW|wx.ALL, 5) - #-- buttons --# - btnSizer = wx.BoxSizer(wx.HORIZONTAL) - - if style & wx.YES_NO == wx.YES_NO: - btnYes = wx.Button(parent, wx.ID_YES, 'Yes') - btnSizer.Add(btnYes, 0, - wx.ALIGN_CENTRE|wx.LEFT|wx.RIGHT, 10) - btnNo = wx.Button(parent, wx.ID_NO, 'No') - btnSizer.Add(btnNo, 0, - wx.ALIGN_CENTRE|wx.LEFT|wx.RIGHT, 10) - if style & wx.YES_DEFAULT == wx.YES_DEFAULT: - btnYes.SetDefault() - elif style & wx.NO_DEFAULT == wx.NO_DEFAULT: - btnNo.SetDefault() - wx.EVT_BUTTON(self, wx.ID_YES, self.OnButton) - wx.EVT_BUTTON(self, wx.ID_NO, self.OnButton) - else: - if style & wx.OK == wx.OK: - btnOK = wx.Button(parent, wx.ID_OK, 'OK') - btnOK.SetDefault() - btnSizer.Add(btnOK, 0, - wx.ALIGN_CENTRE|wx.LEFT|wx.RIGHT, 10) - if style & wx.CANCEL == wx.CANCEL: - btnCancel = wx.Button(parent, wx.ID_CANCEL, 'Cancel') - btnSizer.Add(btnCancel, 0, - wx.ALIGN_CENTRE|wx.LEFT|wx.RIGHT, 10) - - sizer.Add(btnSizer, 0, wx.ALIGN_CENTRE|wx.TOP, 5) - #-- - parent.SetAutoLayout( True ) - parent.SetSizer(sizer ) - sizer.Fit( parent ) - sizer.SetSizeHints( parent ) - -#----------------------------------------------------------------------# -def main(argv): - """ - The default host will be None if the environment variable - PYRO_NS_HOSTNAME is not set. - The default port will be 9090 (Pyro.config.PYRO_NS_PORT) if - PYRO_NS_BC_PORT environment variable is not set. - """ - nsHost = os.getenv('PYRO_NS_HOSTNAME') - nsPort = os.getenv('PYRO_NS_BC_PORT') or Pyro.config.PYRO_NS_PORT - bcAddr = Pyro.config.PYRO_NS_BC_ADDR - if bcAddr: - bcAddr=bcAddr.strip() - bcAddr=bcAddr or None - - class wx_NSCApp(wx.App): - def OnInit(self): - Pyro.core.initClient() - frame = wx_NSC(nsHost, nsPort, bcAddr) - frame.SetSize(wx.Size(630,500)) - frame.Show(True) - return True - - app = wx_NSCApp(0) - app.MainLoop() - -# allow easy usage with python -m -if __name__=="__main__": - main(sys.argv) diff --git a/lib/Pyro/xnsc.py b/lib/Pyro/xnsc.py deleted file mode 100644 index ce97b555474..00000000000 --- a/lib/Pyro/xnsc.py +++ /dev/null @@ -1,350 +0,0 @@ -############################################################################# -# -# Pyro Name Server Control Tool with GUI -# -# This is part of "Pyro" - Python Remote Objects -# which is (c) Irmen de Jong - irmen@razorvine.net -# -############################################################################# - -import sys, time -from Tkinter import * -from Pyro.naming import NameServerLocator -from Pyro.errors import NamingError, ConnectionClosedError -import Pyro.core - -class xnscFrame(object): - - def quit(self): - self.master.quit() - - def clearOutput(self): - self.text_out.delete('1.0',AtEnd()) - self.outputln(time.asctime()) - - def output(self,txt): - self.text_out.insert(AtEnd(),txt) - self.text_out.yview(AtEnd()) - - def outputln(self,txt): - self.output(txt+'\n') - - def b_clearoutput(self, event=None): - self.clearOutput() - - def b_findNS(self,event=None): - self.clearOutput() - hst,prt = None,None - self.authID = self.entry_AuthID.get() - if event: - # Pressed in entry box - addr = self.entry_NSloc.get().split(':') - hst=addr[0] - if len(addr)>1: - prt=int(addr[1]) - # We need to keep the host/port for the shutdown button... - self.NShost = hst - self.NSport = prt - self.outputln('*** finding NS') - locator=NameServerLocator(identification=self.authID) - bcaddr=self.entry_BCAddr.get().strip() or None - try: - self.NS=locator.getNS(hst,prt,trace=1,bcaddr=bcaddr) - self.entry_NSloc.delete(0,AtEnd()) - self.entry_NSloc.insert(AtEnd(),self.NS.URI.address+':'+str(self.NS.URI.port)) - self.entry_AuthID.delete(0,AtEnd()) - self.entry_AuthID.insert(AtEnd(),'****') - - self.enable_buttons() - self.outputln(' found, URI='+str(self.NS.URI)) - except: - self.disable_buttons() - self.outputln(' not found:'); - a,b = sys.exc_info()[:2] - self.outputln(' '+str(a)+' : '+str(b)) - self.outputln('See standard output for trace messages.') - - def handle_comm_error(self,name): - # Handle a communication error: disable buttons and print exception - a,b = sys.exc_info()[:2] - self.outputln('*** '+name+': exception occured:') - self.outputln(' '+str(a)+' : '+str(b)) - if a==ConnectionClosedError: - self.disable_buttons() - self.outputln('*** Connection with NS lost - reconnect') - - def printError(self, msg, exc): - line="## %s: " % msg - if isinstance(exc.args, (list, tuple)): - line+="; ".join(exc.args[:-1]) - else: - line+=exc.args - line+=" ##" - self.outputln(line) - - def b_list(self,event=None): - names = self.entry_arg.get().split() - try: - if names: - self.outputln('*** List groups:') - for n in names: - self.output(' '+self.NS.fullName(n)+' --> ') - try: - self.printList(self.NS.list(n)) - except NamingError,x: - self.printError("can't list",x) - else: - self.outputln('*** List default group:') - self.printList(self.NS.list(None)) - except: - self.handle_comm_error('list') - def printList(self,lst): - out='( ' - lst.sort() - for (n,t) in lst: - if t==0: - out+='['+n+'] ' - elif t==1: - out+=n+' ' - self.outputln(out+')') - - def b_listall(self,event=None): - try: - flat=self.NS.flatlist() - flat.sort() - self.outputln('--------- Flat dump of namespace') - for (name,val) in flat: - self.outputln(' '+name+' --> '+str(val)) - self.outputln('--------- End dump') - except: - self.handle_comm_error('listall') - - def b_register(self,event=None): - self.outputln('*** registering with NS:') - try: - (name,uri) = self.entry_arg.get().split() - try: - self.NS.register(name,uri) - uri=Pyro.core.PyroURI(uri) - self.outputln(' '+name+' --> '+str(uri)) - except NamingError,x: - self.printError("Error from NS", x) - except: - self.handle_comm_error('register') - except ValueError: - self.outputln(' Invalid arguments, use " ".') - - def b_resolve(self,event=None): - self.outputln('*** resolving:') - name=self.entry_arg.get() - if not name: - self.outputln(' Invalid arguments, use "".') - else: - try: - uri=self.NS.resolve(name) - self.outputln(' '+name+' --> '+str(uri)) - except NamingError,x: - self.printError("can't resolve '"+name+"'", x) - except: - self.handle_comm_error('resolve') - - def b_remove(self,event=None): - self.outputln('*** removing:') - name=self.entry_arg.get() - if not name: - self.outputln(' Invalid arguments, use "".') - else: - try: - self.NS.unregister(name) - self.outputln('*** removed: '+name) - except NamingError,x: - self.printError("Can't remove '"+name+"'", x) - except: - self.handle_comm_error('remove') - - def b_ping(self,event=None): - try: - self.NS.ping() - self.outputln('*** ping NS: up and running!') - except: - self.handle_comm_error('ping') - - def b_creategroup(self,event=None): - name=self.entry_arg.get() - if not name: - self.outputln(' Invalid arguments, use "".') - else: - try: - self.NS.createGroup(name) - self.outputln('*** group created: '+name) - except Exception,x: - self.printError("Can't create group",x) - - def b_deletegroup(self,event=None): - name=self.entry_arg.get() - if not name: - self.outputln(' Invalid arguments, use "".') - else: - try: - self.NS.deleteGroup(name) - self.outputln('*** group deleted: '+name) - except Exception,x: - self.printError("Can't delete group",x) - - def b_showmeta(self,event=None): - name=self.NS.fullName(self.entry_arg.get()) - self.outputln('*** showing meta info of: '+name) - try: - self.outputln("system meta info : "+str(self.NS._getSystemMeta(name))) - self.outputln(" user meta info : "+str(self.NS.getMeta(name))) - except NamingError,x: - self.printError("Can't get Meta info",x) - except: - self.handle_comm_error('showmeta') - - def b_setmeta(self,event=None): - self.outputln('*** setting user meta data:') - try: - (name,meta) = self.entry_arg.get().split(None,1) - try: - self.NS.setMeta(name,meta) - self.outputln(' '+name+' META='+meta) - except NamingError,x: - self.printError("Error from NS", x) - except: - self.handle_comm_error('setmeta') - except ValueError: - self.outputln(' Invalid arguments, use " ".') - - def b_resync(self,event=None): - self.outputln("*** resync NS with twin") - try: - self.NS.resync() - except NamingError,x: - self.printError("Can't resync",x) - except: - self.handle_comm_error('resync') - - def b_shutdown(self,event=None): - locator = NameServerLocator(self.authID) - try: - result = locator.sendSysCommand('shutdown',self.NShost,self.NSport,0) - self.outputln('*** The NS replied to the shutdown message: '+str(result)) - except: - self.disable_buttons() - self.outputln(' not found:'); - a,b = sys.exc_info()[:2] - self.outputln(' '+str(a)+' : '+str(b)) - - def enable_buttons(self): - self.enable_disable_buttons(NORMAL) - - def disable_buttons(self): - self.enable_disable_buttons(DISABLED) - - def enable_disable_buttons(self,state): - self.but_ping['state']=state - self.but_list['state']=state - self.but_listall['state']=state - self.but_resolve['state']=state - self.but_register['state']=state - self.but_remove['state']=state - self.but_shutdown['state']=state - self.but_showmeta['state']=state - self.but_setmeta['state']=state - self.but_resync['state']=state - self.but_creategroup['state']=state - self.but_deletegroup['state']=state - - def createWidgets(self): - frame_top = Frame(self.master,borderwidth=2,relief=GROOVE) - frame_top1 = Frame(frame_top,borderwidth=0) - Label(frame_top1,text='Name Server Location (host:port)').pack(side=LEFT,anchor=W) - self.entry_NSloc=Entry(frame_top1) - self.entry_NSloc.bind('',self.b_findNS) - self.entry_NSloc.pack(expand=1,fill=X,side=LEFT) - Label(frame_top1,text='(press enter)').pack(side=LEFT,anchor=W) - frame_top1.pack(fill=X) - frame_top2 = Frame(frame_top,borderwidth=0) - frame_top3 = Frame(frame_top,borderwidth=0) - Label(frame_top2,text='Authorization ID:').pack(side=LEFT,anchor=W) - self.entry_AuthID=Entry(frame_top2) - self.entry_AuthID.bind('',self.b_findNS) - self.entry_AuthID.pack(expand=1,fill=X,side=LEFT) - Label(frame_top3,text='Broadcast address:').pack(side=LEFT,anchor=W) - self.entry_BCAddr=Entry(frame_top3) - self.entry_BCAddr.pack(expand=1,fill=X,side=LEFT) - self.but_findNS=Button(frame_top3,text='Auto Discover NS',command=self.b_findNS) - self.QUIT=Button(frame_top3,text='QUIT',command=self.quit) - self.QUIT.pack(side=RIGHT) - self.but_findNS.pack(side=RIGHT) - frame_top2.pack(fill=X) - frame_top3.pack(fill=X) - frame_top.pack(fill=X) - - frame_cmds=Frame(self.master) - frame_cmds1=Frame(frame_cmds) - frame_cmds2=Frame(frame_cmds) - self.but_ping=Button(frame_cmds1,text='Ping',state=DISABLED,command=self.b_ping) - self.but_list=Button(frame_cmds1,text='List',state=DISABLED,command=self.b_list) - self.but_listall=Button(frame_cmds1,text='List All',state=DISABLED,command=self.b_listall) - self.but_register=Button(frame_cmds2,text='Register',state=DISABLED,command=self.b_register) - self.but_resolve=Button(frame_cmds1,text='Resolve',state=DISABLED,command=self.b_resolve) - self.but_remove=Button(frame_cmds2,text='Remove',state=DISABLED,command=self.b_remove) - self.but_creategroup=Button(frame_cmds2,text='Create Group',state=DISABLED,command=self.b_creategroup) - self.but_deletegroup=Button(frame_cmds2,text='Delete Group',state=DISABLED,command=self.b_deletegroup) - self.but_showmeta=Button(frame_cmds1,text='Show Meta',state=DISABLED,command=self.b_showmeta) - self.but_setmeta=Button(frame_cmds1,text='Set Meta',state=DISABLED,command=self.b_setmeta) - self.but_resync=Button(frame_cmds1,text='ReSync',state=DISABLED,command=self.b_resync) - self.but_shutdown=Button(frame_cmds1,text='Shutdown',state=DISABLED,command=self.b_shutdown) - self.but_clearoutput=Button(frame_cmds2,text='Clear output',command=self.b_clearoutput) - Label(frame_cmds,text='NS commands:').pack(side=LEFT) - self.but_ping.pack(side=LEFT) - self.but_list.pack(side=LEFT) - self.but_listall.pack(side=LEFT) - self.but_register.pack(side=LEFT) - self.but_resolve.pack(side=LEFT) - self.but_remove.pack(side=LEFT) - self.but_creategroup.pack(side=LEFT) - self.but_deletegroup.pack(side=LEFT) - self.but_showmeta.pack(side=LEFT) - self.but_setmeta.pack(side=LEFT) - self.but_resync.pack(side=LEFT) - self.but_shutdown.pack(side=LEFT) - self.but_clearoutput.pack(side=RIGHT) - - frame_args=Frame(self.master,borderwidth=2) - self.entry_arg=Entry(frame_args) - Label(frame_args,text='Command arguments').pack(side=LEFT) - self.entry_arg.pack(expand=1,fill=X) - - frame_output=Frame(self.master) - ys=Scrollbar(frame_output,orient=VERTICAL) - self.text_out=Text(frame_output,yscrollcommand=ys.set,width=90,height=20) - ys['command']=self.text_out.yview - ys.pack(fill=Y,side=LEFT) - self.text_out.pack(side=LEFT,expand=1,fill=BOTH) - - # pack root children: - frame_cmds1.pack(fill=X) - frame_cmds2.pack(fill=X) - frame_cmds.pack(fill=X) - frame_args.pack(fill=X) - frame_output.pack(fill=BOTH,expand=1) - - def __init__(self, master=None): - self.master = master - self.createWidgets() - -def main(argv): - Pyro.core.initClient() - root=Tk() - root.title('xnsc - Pyro Name Server control tool - Pyro version '+Pyro.constants.VERSION) - app=xnscFrame(root) - root.protocol('WM_DELETE_WINDOW',root.quit) - root.mainloop() - -# allow easy usage with python -m -if __name__=="__main__": - import sys - main(sys.argv) diff --git a/lib/cylc/CylcOptionParsers.py b/lib/cylc/CylcOptionParsers.py index f4744ca6891..0c9f5316ad6 100644 --- a/lib/cylc/CylcOptionParsers.py +++ b/lib/cylc/CylcOptionParsers.py @@ -420,6 +420,5 @@ def parse_multitask_compat(cls, options, mtask_args): # Element 1 may be a regular expression, so it may contain "." but # should not contain a "/". # All other elements should contain no "." and "/". - return (mtask_args[0], mtask_args[1]) - else: - return (mtask_args, None) + return (mtask_args[0] + "." + mtask_args[1],) + return mtask_args diff --git a/lib/cylc/broadcast_report.py b/lib/cylc/broadcast_report.py index 80760b32ddd..1872bb413cc 100644 --- a/lib/cylc/broadcast_report.py +++ b/lib/cylc/broadcast_report.py @@ -39,7 +39,7 @@ def get_broadcast_bad_options_report(bad_options, is_set=False): msg = BAD_OPTIONS_TITLE for key, values in sorted(bad_options.items()): for value in values: - if isinstance(value, tuple): + if isinstance(value, tuple) or isinstance(value, list): value_str = "" values = list(value) while values: diff --git a/lib/cylc/cfgspec/globalcfg.py b/lib/cylc/cfgspec/globalcfg.py index 6deebd3eb08..2e3d748688c 100644 --- a/lib/cylc/cfgspec/globalcfg.py +++ b/lib/cylc/cfgspec/globalcfg.py @@ -129,8 +129,17 @@ 'gui': vdr(vtype='string', default="gvim -f"), }, + 'communication': { + 'method': vdr(vtype='string', default="https", + options=["https", "hmac"]), + 'base port': vdr(vtype='integer', default=43001), + 'maximum number of ports': vdr(vtype='integer', default=100), + 'ports directory': vdr(vtype='string', default="$HOME/.cylc/ports/"), + 'proxies_on': vdr(vtype='boolean', default=False) + }, + 'pyro': { - 'base port': vdr(vtype='integer', default=7766), + 'base port': vdr(vtype='integer', default=43001), 'maximum number of ports': vdr(vtype='integer', default=100), 'ports directory': vdr(vtype='string', default="$HOME/.cylc/ports/"), }, @@ -147,7 +156,7 @@ 'work directory': vdr(vtype='string', default="$HOME/cylc-run"), 'task communication method': vdr( vtype='string', - options=["pyro", "ssh", "poll"], default="pyro"), + options=["default", "ssh", "poll"], default="default"), 'remote copy template': vdr( vtype='string', default='scp -oBatchMode=yes -oConnectTimeout=10'), @@ -197,7 +206,7 @@ 'run directory': vdr(vtype='string'), 'work directory': vdr(vtype='string'), 'task communication method': vdr( - vtype='string', options=["pyro", "ssh", "poll"]), + vtype='string', options=["default", "ssh", "poll"]), 'remote copy template': vdr(vtype='string'), 'remote shell template': vdr(vtype='string'), 'use login shell': vdr(vtype='boolean'), @@ -542,8 +551,8 @@ def create_cylc_run_tree(self, suite): if value: self.create_directory(value, item) - item = '[pyro]ports directory' - value = cfg['pyro']['ports directory'] + item = '[communication]ports directory' + value = cfg['communication']['ports directory'] self.create_directory(value, item) def get_tmpdir(self): @@ -588,8 +597,8 @@ def transform(self): for key, val in cfg['documentation']['files'].items(): cfg['documentation']['files'][key] = expandvars(val) - cfg['pyro']['ports directory'] = expandvars( - cfg['pyro']['ports directory']) + cfg['communication']['ports directory'] = expandvars( + cfg['communication']['ports directory']) for key, val in cfg['hosts']['localhost'].items(): if val and 'directory' in key: diff --git a/lib/cylc/config.py b/lib/cylc/config.py index 58b40a049ba..f3b33bc7fee 100644 --- a/lib/cylc/config.py +++ b/lib/cylc/config.py @@ -1781,11 +1781,15 @@ def get_conditional_label(self, expression): return label def get_graph_raw(self, start_point_string, stop_point_string, - group_nodes=[], ungroup_nodes=[], + group_nodes=None, ungroup_nodes=None, ungroup_recursive=False, group_all=False, ungroup_all=False): """Convert the abstract graph edges held in self.edges (etc.) to actual edges for a concrete range of cycle points.""" + if group_nodes is None: + group_nodes = [] + if ungroup_nodes is None: + ungroup_nodes = [] members = self.runtime['first-parent descendants'] hierarchy = self.runtime['first-parent ancestors'] diff --git a/lib/cylc/gui/app_gcylc.py b/lib/cylc/gui/app_gcylc.py index a0e8f9b252e..07bb69154e6 100644 --- a/lib/cylc/gui/app_gcylc.py +++ b/lib/cylc/gui/app_gcylc.py @@ -136,17 +136,17 @@ class InitData(object): Class to hold initialisation data. """ def __init__(self, suite, owner, host, port, db, - pyro_timeout, template_vars, template_vars_file, + comms_timeout, template_vars, template_vars_file, ungrouped_views, use_defn_order): self.suite = suite self.owner = owner self.host = host self.port = port self.db = db - if pyro_timeout: - self.pyro_timeout = float(pyro_timeout) + if comms_timeout: + self.comms_timeout = float(comms_timeout) else: - self.pyro_timeout = None + self.comms_timeout = None self.template_vars_opts = "" for tv in template_vars: @@ -527,7 +527,7 @@ class ControlApp(object): VIEWS["graph"] = ControlGraph VIEWS_ORDERED.append("graph") - def __init__(self, suite, db, owner, host, port, pyro_timeout, + def __init__(self, suite, db, owner, host, port, comms_timeout, template_vars, template_vars_file, restricted_display): gobject.threads_init() @@ -540,7 +540,7 @@ def __init__(self, suite, db, owner, host, port, pyro_timeout, if "graph" in self.__class__.VIEWS_ORDERED: self.__class__.VIEWS_ORDERED.remove('graph') - self.cfg = InitData(suite, owner, host, port, db, pyro_timeout, + self.cfg = InitData(suite, owner, host, port, db, comms_timeout, template_vars, template_vars_file, gcfg.get(["ungrouped views"]), gcfg.get(["sort by definition order"])) @@ -981,7 +981,7 @@ def click_exit(self, foo): def click_open(self, foo=None): app = dbchooser(self.window, self.cfg.db, self.cfg.owner, - self.cfg.cylc_tmpdir, self.cfg.pyro_timeout) + self.cfg.cylc_tmpdir, self.cfg.comms_timeout) reg, auth = None, None while True: response = app.window.run() @@ -1000,16 +1000,16 @@ def click_open(self, foo=None): self.reset(reg, auth) def pause_suite(self, bt): - self.put_pyro_command('hold_suite') + self.put_comms_command('hold_suite') def resume_suite(self, bt): - self.put_pyro_command('release_suite') + self.put_comms_command('release_suite') def stopsuite_default(self, *args): """Try to stop the suite (after currently running tasks...).""" if not self.get_confirmation("Stop suite %s?" % self.cfg.suite): return - self.put_pyro_command('set_stop_cleanly') + self.put_comms_command('set_stop_cleanly') def stopsuite(self, bt, window, kill_rb, stop_rb, stopat_rb, stopct_rb, stoptt_rb, stopnow_rb, stoppoint_entry, stopclock_entry, @@ -1074,17 +1074,22 @@ def stopsuite(self, bt, window, kill_rb, stop_rb, stopat_rb, stopct_rb, window.destroy() if stop: - self.put_pyro_command('set_stop_cleanly', False) + self.put_comms_command('set_stop_cleanly', + kill_active_tasks=False) elif stopkill: - self.put_pyro_command('set_stop_cleanly', True) + self.put_comms_command('set_stop_cleanly', + kill_active_tasks=True) elif stopat: - self.put_pyro_command('set_stop_after_point', stop_point_string) + self.put_comms_command('set_stop_after_point', + point_string=stop_point_string) elif stopnow: - self.put_pyro_command('stop_now') + self.put_comms_command('stop_now') elif stopclock: - self.put_pyro_command('set_stop_after_clock_time', stopclock_time) + self.put_comms_command('set_stop_after_clock_time', + datetime_string=stopclock_time) elif stoptask: - self.put_pyro_command('set_stop_after_task', stoptask_id) + self.put_comms_command('set_stop_after_task', + task_id=stoptask_id) def load_point_strings(self, bt, startentry, stopentry): item1 = " -i '[scheduling]initial cycle point'" @@ -1533,7 +1538,7 @@ def change_runahead(self, w, entry, window): else: limit = ent window.destroy() - self.put_pyro_command('set_runahead', limit) + self.put_comms_command('set_runahead', interval=limit) def update_tb(self, tb, line, tags=None): if tags: @@ -1543,7 +1548,8 @@ def update_tb(self, tb, line, tags=None): def popup_requisites(self, w, e, task_id): name, point_string = TaskID.split(task_id) - result = self.get_pyro_info('get_task_requisites', name, point_string) + result = self.get_comms_info( + 'get_task_requisites', name=name, point_string=point_string) if result: # (else no tasks were found at all -suite shutting down) if task_id not in result: @@ -1653,12 +1659,12 @@ def hold_task(self, b, task_ids, stop=True, is_family=False): for task_id in task_ids: if not self.get_confirmation("Hold %s?" % (task_id)): return - self.put_pyro_command('hold_task', task_ids) + self.put_comms_command('hold_tasks', items=task_ids) else: for task_id in task_ids: if not self.get_confirmation("Release %s?" % (task_id)): return - self.put_pyro_command('release_task', task_ids) + self.put_comms_command('release_task', items=task_ids) def trigger_task_now(self, b, task_ids, is_family=False): """Trigger task via the suite daemon's command interface.""" @@ -1668,7 +1674,7 @@ def trigger_task_now(self, b, task_ids, is_family=False): for task_id in task_ids: if not self.get_confirmation("Trigger %s?" % task_id): return - self.put_pyro_command('trigger_task', task_ids) + self.put_comms_command('trigger_tasks', items=task_ids) def trigger_task_edit_run(self, b, task_id): """ @@ -1690,7 +1696,7 @@ def poll_task(self, b, task_ids, is_family=False): for task_id in task_ids: if not self.get_confirmation("Poll %s?" % task_id): return - self.put_pyro_command('poll_tasks', task_ids) + self.put_comms_command('poll_tasks', items=task_ids) def kill_task(self, b, task_ids, is_family=False): """Kill a task/family.""" @@ -1701,7 +1707,7 @@ def kill_task(self, b, task_ids, is_family=False): if not self.get_confirmation("Kill %s?" % task_id, force_prompt=True): return - self.put_pyro_command('kill_tasks', task_ids) + self.put_comms_command('kill_tasks', items=task_ids) def spawn_task(self, b, e, task_ids, is_family=False): """For tasks to spawn their successors.""" @@ -1713,7 +1719,7 @@ def spawn_task(self, b, e, task_ids, is_family=False): return False if not self.get_confirmation("Force spawn %s?" % task_id): return - self.put_pyro_command('spawn_tasks', task_ids, None) + self.put_comms_command('spawn_tasks', items=task_ids) def reset_task_state(self, b, e, task_ids, state, is_family=False): """Reset the state of a task/family.""" @@ -1725,7 +1731,8 @@ def reset_task_state(self, b, e, task_ids, state, is_family=False): return False if not self.get_confirmation("reset %s to %s?" % (task_id, state)): return - self.put_pyro_command('reset_task_state', task_ids, None, state) + self.put_comms_command('reset_task_state', items=task_ids, + state=state) def remove_task(self, b, task_ids, is_family): """Remove a task.""" @@ -1737,7 +1744,7 @@ def remove_task(self, b, task_ids, is_family): "Remove %s after spawning?" % task_id): return name, point_string = TaskID.split(task_id) - self.put_pyro_command('remove_task', task_ids, None, True) + self.put_comms_command('remove_task', task_ids, spawn=True) def remove_task_nospawn(self, b, task_ids, is_family=False): """Remove a task, without spawn.""" @@ -1748,7 +1755,7 @@ def remove_task_nospawn(self, b, task_ids, is_family=False): if not self.get_confirmation( "Remove %s without spawning?" % task_id): return - self.put_pyro_command('remove_task', task_ids, None, False) + self.put_comms_command('remove_tasks', task_ids, spawn=False) def stopsuite_popup(self, b): window = gtk.Window() @@ -2167,24 +2174,26 @@ def insert_task(self, w, window, entry_task_ids, entry_stop_point): window.destroy() if not stop_point_str.strip(): stop_point_str = None - self.put_pyro_command( - 'insert_task', task_ids, None, None, stop_point_str) + self.put_comms_command( + 'insert_tasks', items=task_ids, + stop_point_string=stop_point_str + ) def poll_all(self, w): """Poll all active tasks.""" if not self.get_confirmation("Poll all submitted/running task jobs?"): return - self.put_pyro_command('poll_tasks', None, None, None) + self.put_comms_command('poll_tasks') def reload_suite(self, w): if not self.get_confirmation("Reload suite definition?"): return - self.put_pyro_command('reload_suite') + self.put_comms_command('reload_suite') def nudge_suite(self, w): if not self.get_confirmation("Nudge suite?"): return - self.put_pyro_command('nudge') + self.put_comms_command('nudge') def _popup_logview(self, task_id, task_state_summary, choice=None): """Display task job log files in a combo log viewer.""" @@ -2209,7 +2218,7 @@ def _popup_logview(self, task_id, task_state_summary, choice=None): ) for submit_num, job_user_at_host in sorted( job_hosts.items(), reverse=True): - submit_num_str = "%02d" % submit_num + submit_num_str = "%02d" % int(submit_num) local_job_log_dir = os.path.join(itask_log_dir, submit_num_str) for filename in ["job", "job-activity.log"]: filenames.append(os.path.join(local_job_log_dir, filename)) @@ -2794,7 +2803,7 @@ def describe_suite(self, w): # Show suite title and description. if self.updater.connected: # Interrogate the suite daemon. - info = self.get_pyro_info('get_suite_info') + info = self.get_comms_info('get_suite_info') descr = '\n'.join( "%s: %s" % (key, val) for key, val in info.items()) info_dialog(descr, self.window).inform() @@ -3314,19 +3323,19 @@ def destroy_theme_legend(self, widget): """Handle a destroy of the theme legend window.""" self.theme_legend_window = None - def put_pyro_command(self, command, *args): + def put_comms_command(self, command, **kwargs): try: success, msg = self.updater.suite_command_client.put_command( - command, *args) + command, **kwargs) except Exception, x: warning_dialog(x.__str__(), self.window).warn() else: if not success: warning_dialog(msg, self.window).warn() - def get_pyro_info(self, command, *args): + def get_comms_info(self, command, **kwargs): try: - return self.updater.suite_info_client.get_info(command, *args) + return self.updater.suite_info_client.get_info(command, **kwargs) except Exception as exc: warning_dialog(str(exc), self.window).warn() diff --git a/lib/cylc/gui/updater.py b/lib/cylc/gui/updater.py index 7caaa4e6761..6fd0b75a68e 100644 --- a/lib/cylc/gui/updater.py +++ b/lib/cylc/gui/updater.py @@ -19,7 +19,6 @@ import re import sys -import Pyro import atexit import gobject import threading @@ -29,6 +28,7 @@ from cylc.exceptions import PortFileError import cylc.flags from cylc.dump import get_stop_state_summary +from cylc.network import ConnectionDeniedError from cylc.network.suite_state import ( StateSummaryClient, SuiteStillInitialisingError, get_suite_status_string, SUITE_STATUS_NOT_CONNECTED, SUITE_STATUS_CONNECTED, @@ -174,7 +174,7 @@ def __init__(self, app): self.version_mismatch_warned = False client_args = (self.cfg.suite, self.cfg.owner, self.cfg.host, - self.cfg.pyro_timeout, self.cfg.port, self.cfg.db, + self.cfg.comms_timeout, self.cfg.port, self.cfg.db, self.cfg.my_uuid) self.state_summary_client = StateSummaryClient(*client_args) self.suite_info_client = SuiteInfoClient(*client_args) @@ -187,7 +187,7 @@ def reconnect(self): """Try to reconnect to the suite daemon.""" if cylc.flags.debug: print >> sys.stderr, " reconnection...", - # Reset Pyro clients. + # Reset comms clients. self.suite_log_client.reset() self.state_summary_client.reset() self.suite_info_client.reset() @@ -195,10 +195,6 @@ def reconnect(self): try: self.daemon_version = self.suite_info_client.get_info( 'get_cylc_version') - except KeyError: - self.daemon_version = "??? (pre 6.1.2?)" - if cylc.flags.debug: - print >> sys.stderr, "succeeded (old daemon)" except PortFileError as exc: if cylc.flags.debug: traceback.print_exc() @@ -218,16 +214,12 @@ def reconnect(self): self.info_bar.set_update_time( None, self.info_bar.DISCONNECTED_TEXT) return - except Pyro.errors.NamingError as exc: - if cylc.flags.debug: - traceback.print_exc() - return except Exception as exc: if cylc.flags.debug: traceback.print_exc() if not self.connect_fail_warned: self.connect_fail_warned = True - if isinstance(exc, Pyro.errors.ConnectionDeniedError): + if isinstance(exc, ConnectionDeniedError): gobject.idle_add( self.warn, "ERROR: %s\n\nIncorrect suite passphrase?" % exc) @@ -276,15 +268,10 @@ def set_update(self, should_update): def retrieve_err_log(self): """Retrieve suite err log; return True if it has changed.""" - try: - new_err_content, new_err_size = ( - self.suite_log_client.get_err_content( - self.err_log_size, self._err_num_log_lines)) - except AttributeError: - # TODO: post-backwards compatibility concerns, remove this handling - new_err_content = "" - new_err_size = self.err_log_size - + new_err_content, new_err_size = ( + self.suite_log_client.get_err_content( + self.err_log_size, self._err_num_log_lines) + ) err_log_changed = (new_err_size != self.err_log_size) if err_log_changed: self.err_log_lines += new_err_content.splitlines() @@ -294,30 +281,25 @@ def retrieve_err_log(self): def retrieve_summary_update_time(self): """Retrieve suite summary update time; return True if changed.""" - do_update = False - try: - summary_update_time = ( - self.state_summary_client.get_suite_state_summary_update_time() - ) - if (summary_update_time is None or - self._summary_update_time is None or - summary_update_time != self._summary_update_time): - self._summary_update_time = summary_update_time - do_update = True - except AttributeError: - # TODO: post-backwards compatibility concerns, remove this handling - # Force an update for daemons using the old API - do_update = True - return do_update + summary_update_time = float( + self.state_summary_client.get_suite_state_summary_update_time() + ) + if (summary_update_time is None or + self._summary_update_time is None or + summary_update_time != self._summary_update_time): + self._summary_update_time = summary_update_time + return True + return False def retrieve_state_summaries(self): """Retrieve suite summary.""" + ret = self.state_summary_client.get_suite_state_summary() glbl, states, fam_states = ( self.state_summary_client.get_suite_state_summary()) self.ancestors = self.suite_info_client.get_info( 'get_first_parent_ancestors') self.ancestors_pruned = self.suite_info_client.get_info( - 'get_first_parent_ancestors', True) + 'get_first_parent_ancestors', pruned=True) self.descendants = self.suite_info_client.get_info( 'get_first_parent_descendants') self.all_families = self.suite_info_client.get_info('get_all_families') @@ -326,18 +308,14 @@ def retrieve_state_summaries(self): self.mode = glbl['run_mode'] - if self.cfg.use_defn_order and 'namespace definition order' in glbl: - # (protect for compat with old suite daemons) + if self.cfg.use_defn_order: nsdo = glbl['namespace definition order'] if self.ns_defn_order != nsdo: self.ns_defn_order = nsdo self.dict_ns_defn_order = dict(zip(nsdo, range(0, len(nsdo)))) - try: - self.update_time_str = get_time_string_from_unix_time( - glbl['last_updated']) - except (TypeError, ValueError): - # Older suite... - self.update_time_str = glbl['last_updated'].isoformat() + + self.update_time_str = get_time_string_from_unix_time( + glbl['last_updated']) self.global_summary = glbl if self.restricted_display: @@ -347,19 +325,8 @@ def retrieve_state_summaries(self): self.full_fam_state_summary = fam_states self.refilter() - try: - self.status = glbl['status_string'] - except KeyError: - # Back compat for suite daemons <= 6.9.1. - self.status = get_suite_status_string( - glbl['paused'], glbl['stopping'], glbl['will_pause_at'], - glbl['will_stop_at']) - - try: - self.is_reloading = glbl['reloading'] - except KeyError: - # Back compat. - pass + self.status = glbl['status_string'] + self.is_reloading = glbl['reloading'] def set_stopped(self): """Reset data and clients when suite is stopped.""" @@ -437,29 +404,6 @@ def update(self): self.info_bar.prog_bar_start, SUITE_STATUS_INITIALISING) self.info_bar.set_state([]) return False - except Pyro.errors.NamingError as exc: - if self.daemon_version is not None: - # Back compat <= 6.4.0 the state summary object was not - # connected to Pyro until initialisation was completed. - if cylc.flags.debug: - print >> sys.stderr, ( - " daemon <= 6.4.0, suite initializing ...") - self.set_status(SUITE_STATUS_INITIALISING) - if self.info_bar.prog_bar_can_start(): - gobject.idle_add(self.info_bar.prog_bar_start, - SUITE_STATUS_INITIALISING) - self.info_bar.set_state([]) - # Reconnect till we get the suite state object. - self.reconnect() - return False - else: - if cylc.flags.debug: - print >> sys.stderr, " CONNECTION LOST", str(exc) - self.set_stopped() - if self.info_bar.prog_bar_active(): - gobject.idle_add(self.info_bar.prog_bar_stop) - self.reconnect() - return False except Exception as exc: if self.status == SUITE_STATUS_STOPPING: # Expected stop: prevent the reconnection warning dialog. diff --git a/lib/cylc/gui/updater_graph.py b/lib/cylc/gui/updater_graph.py index 50ec1cdb113..573c1835f8a 100644 --- a/lib/cylc/gui/updater_graph.py +++ b/lib/cylc/gui/updater_graph.py @@ -302,38 +302,33 @@ def update_graph(self): oldest = self.oldest_point_string newest = self.newest_point_string + group_for_server = self.group + if self.group == []: + group_for_server = None + + ungroup_for_server = self.ungroup + if self.ungroup == []: + ungroup_for_server = None + try: res = self.updater.suite_info_client.get_info( - 'get_graph_raw', oldest, newest, self.group, self.ungroup, - self.ungroup_recursive, self.group_all, self.ungroup_all) - except TypeError: - # Back compat with pre cylc-6 suite daemons. - res = self.updater.suite_info_client.get( - 'get_graph_raw', oldest, newest, False, self.group, - self.ungroup, self.ungroup_recursive, self.group_all, - self.ungroup_all) + 'get_graph_raw', start_point_string=oldest, + stop_point_string=newest, + group_nodes=group_for_server, + ungroup_nodes=ungroup_for_server, + ungroup_recursive=self.ungroup_recursive, + group_all=self.group_all, + ungroup_all=self.ungroup_all + ) except Exception as exc: # PyroError? print >> sys.stderr, str(exc) return False - # backward compatibility for old suite daemons still running - self.have_leaves_and_feet = False - if isinstance(res, list): - # prior to suite-polling tasks in 5.4.0 - gr_edges = res - suite_polling_tasks = [] - self.leaves = [] - self.feet = [] - else: - if len(res) == 2: - # prior to graph view grouping fix in 5.4.2 - gr_edges, suite_polling_tasks = res - self.leaves = [] - self.feet = [] - elif len(res) == 4: - # 5.4.2 and later - self.have_leaves_and_feet = True - gr_edges, suite_polling_tasks, self.leaves, self.feet = res + print "Res", res + + self.have_leaves_and_feet = True + gr_edges, suite_polling_tasks, self.leaves, self.feet = res + gr_edges = [tuple(edge) for edge in gr_edges] current_id = self.get_graph_id(gr_edges) needs_redraw = current_id != self.prev_graph_id @@ -458,8 +453,10 @@ def update_graph(self): def get_graph_id(self, edges): """If any of these quantities change, the graph should be redrawn.""" + print "Get graph id", edges node_ids = set() for edge in edges: + print " edge", edge node_ids.add(edge[0]) node_ids.add(edge[1]) # Get a set of ids that are actually present in the state summaries. diff --git a/lib/cylc/network/__init__.py b/lib/cylc/network/__init__.py index c6f7ce704da..4cdf72fc963 100644 --- a/lib/cylc/network/__init__.py +++ b/lib/cylc/network/__init__.py @@ -24,13 +24,13 @@ # Names for network-connected objects. # WARNING: these names are don't have consistent formatting, but changing them # will break backward compatibility with older cylc clients! -PYRO_SUITEID_OBJ_NAME = 'cylcid' -PYRO_EXT_TRIG_OBJ_NAME = 'ext-trigger-interface' -PYRO_BCAST_OBJ_NAME = 'broadcast_receiver' -PYRO_CMD_OBJ_NAME = 'command-interface' -PYRO_INFO_OBJ_NAME = 'suite-info' +PYRO_SUITEID_OBJ_NAME = 'id' +PYRO_EXT_TRIG_OBJ_NAME = 'ext-trigger' +PYRO_BCAST_OBJ_NAME = 'broadcast' +PYRO_CMD_OBJ_NAME = 'command' +PYRO_INFO_OBJ_NAME = 'info' PYRO_LOG_OBJ_NAME = 'log' -PYRO_STATE_OBJ_NAME = 'state_summary' +PYRO_STATE_OBJ_NAME = 'state' # Ordered privilege levels for authenticated users. PRIVILEGE_LEVELS = [ @@ -49,6 +49,16 @@ NO_PASSPHRASE = 'the quick brown fox' +class ConnectionDeniedError(Exception): + + """An error raised when the client is not permitted to connect.""" + + MESSAGE = "Not authorized: %s: %s: access type '%s'" + + def __str__(self): + return self.MESSAGE % (self.args[0], self.args[1], self.args[2]) + + def access_priv_ok(server_obj, required_privilege_level): """Return True if a client is allowed access to info from server_obj. @@ -59,9 +69,10 @@ def access_priv_ok(server_obj, required_privilege_level): if threading.current_thread().__class__.__name__ == '_MainThread': # Server methods may be called internally as well as by clients. return True - caller = server_obj.getLocalStorage().caller - client_privilege_level = caller.privilege_level - return (PRIVILEGE_LEVELS.index(client_privilege_level) >= + import cherrypy + user = cherrypy.request.login + priv_level = get_priv_level(user) + return (PRIVILEGE_LEVELS.index(priv_level) >= PRIVILEGE_LEVELS.index(required_privilege_level)) @@ -74,13 +85,65 @@ def check_access_priv(server_obj, required_privilege_level): if threading.current_thread().__class__.__name__ == '_MainThread': # Server methods may be called internally as well as by clients. return - caller = server_obj.getLocalStorage().caller - client_privilege_level = caller.privilege_level - if not (PRIVILEGE_LEVELS.index(client_privilege_level) >= + auth_user, prog_name, user, host, uuid, priv_level = get_client_info() + if not (PRIVILEGE_LEVELS.index(priv_level) >= PRIVILEGE_LEVELS.index(required_privilege_level)): err = CONNECT_DENIED_PRIV_TMPL % ( - client_privilege_level, required_privilege_level, - caller.user, caller.host, caller.prog_name, caller.uuid) + priv_level, required_privilege_level, + user, host, prog_name, uuid + ) getLogger("main").warn(err) # Raise an exception to be sent back to the client. raise Exception(err) + + +def get_client_info(): + """Return information about the most recent cherrypy request, if any.""" + import cherrypy + import uuid + auth_user = cherrypy.request.login + info = cherrypy.request.headers + origin_string = info.get("User-Agent", "") + origin_props = {} + if origin_string: + origin_props = dict( + [_.split("/", 1) for _ in origin_string.split()] + ) + prog_name = origin_props.get("prog_name", "Unknown") + uuid = origin_props.get("uuid", uuid.uuid4()) + if info.get("From"): + user, host = info["From"].split("@") + else: + user, host = ("Unknown", "Unknown") + priv_level = get_priv_level(auth_user) + return auth_user, prog_name, user, host, uuid, priv_level + + +def get_client_connection_denied(): + """Return whether a connection was denied.""" + import cherrypy + if "Authorization" not in cherrypy.request.headers: + # Probably just the initial HTTPS handshake. + return False + status = cherrypy.response.status + if isinstance(status, basestring): + return cherrypy.response.status.split()[0] in ["401", "403"] + return cherrypy.response.status in [401, 403] + + +def get_priv_level(user): + """Get the privilege level for this authenticated user.""" + if user == "cylc": + return PRIVILEGE_LEVELS[-1] + from cylc.config import SuiteConfig + config = SuiteConfig.get_inst() + return config.cfg['cylc']['authentication']['public'] + + +def handle_proxies(): + """Unset proxies if the configuration matches this.""" + from cylc.cfgspec.globalcfg import GLOBAL_CFG + if not GLOBAL_CFG.get(['communication', 'proxies_on']): + import os + os.environ.pop("http_proxy", None) + os.environ.pop("https_proxy", None) diff --git a/lib/cylc/network/connection_validator.py b/lib/cylc/network/connection_validator.py deleted file mode 100644 index 77da440083b..00000000000 --- a/lib/cylc/network/connection_validator.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2016 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import Pyro.core -import hashlib -import logging -import os -import sys - -from Pyro.protocol import DefaultConnValidator -import Pyro.constants -import Pyro.errors -import hmac - -from cylc.cfgspec.globalcfg import GLOBAL_CFG -from cylc.network import NO_PASSPHRASE, PRIVILEGE_LEVELS -from cylc.config import SuiteConfig -from cylc.suite_host import get_hostname, is_remote_host -from cylc.owner import USER - - -# Access for users without the suite passphrase: encrypting the "no passphrase" -# passphrase is unnecessary, but doing so allows common passphrase handling. - -OK_HASHES = GLOBAL_CFG.get()['authentication']['hashes'] -SCAN_HASH = GLOBAL_CFG.get()['authentication']['scan hash'] -if SCAN_HASH not in OK_HASHES: - OK_HASHES.append(SCAN_HASH) - - -CONNECT_DENIED_TMPL = "[client-connect] DENIED %s@%s:%s %s" -CONNECT_ALLOWED_TMPL = "[client-connect] %s@%s:%s privilege='%s' %s" - - -class ConnValidator(DefaultConnValidator): - """Custom Pyro connection validator for user authentication.""" - - HASHES = {} - LENGTH_HASH_DIGESTS = {} - NO_PASSPHRASE_HASHES = {} - - def set_pphrase(self, pphrase): - """Store encrypted suite passphrase (called by the server).""" - self.pphrase_hashes = {} - for hash_name in OK_HASHES: - hash_ = self._get_hash(hash_name) - self.pphrase_hashes[hash_name] = hash_(pphrase).hexdigest() - - def set_default_hash(self, hash_name): - """Configure a hash to use as the default.""" - self._default_hash_name = hash_name - if None in self.HASHES: - self.HASHES.pop(None) # Pop default setting. - - def acceptIdentification(self, daemon, connection, token, challenge): - """Authorize client login.""" - - logger = logging.getLogger('main') - is_old_client = False - # Processes the token returned by createAuthToken. - try: - user, host, uuid, prog_name, proc_passwd = token.split(':', 4) - except ValueError: - # Back compat for old suite client (passphrase only) - # (Allows old scan to see new suites.) - proc_passwd = token - is_old_client = True - user = "(user)" - host = "(host)" - uuid = "(uuid)" - prog_name = "(OLD_CLIENT)" - - hash_name = self._get_hash_name_from_digest_length(proc_passwd) - - if hash_name not in OK_HASHES: - return (0, Pyro.constants.DENIED_SECURITY) - - hash_ = self._get_hash(hash_name) - - # Access for users without the suite passphrase: encrypting the - # no-passphrase is unnecessary, but doing so allows common handling. - no_passphrase_hash = self._get_no_passphrase_hash(hash_name) - - # Check username and password, and set privilege level accordingly. - # The auth token has a binary hash that needs conversion to ASCII. - if self._compare_hmacs( - hmac.new(challenge, - self.pphrase_hashes[hash_name].decode("hex"), - hash_).digest(), - proc_passwd): - # The client has the suite passphrase. - # Access granted at highest privilege level. - priv_level = PRIVILEGE_LEVELS[-1] - elif not is_old_client and self._compare_hmacs( - hmac.new(challenge, - no_passphrase_hash.decode("hex"), - hash_).digest(), - proc_passwd): - # The client does not have the suite passphrase. - # Public access granted at level determined by global/suite config. - config = SuiteConfig.get_inst() - priv_level = config.cfg['cylc']['authentication']['public'] - else: - # Access denied. - if not is_old_client: - # Avoid logging large numbers of denials from old scan clients - # that try all passphrases available to them. - logger.warn(CONNECT_DENIED_TMPL % ( - user, host, prog_name, uuid)) - return (0, Pyro.constants.DENIED_SECURITY) - - # Store client details for use in the connection thread. - connection.user = user - connection.host = host - connection.prog_name = prog_name - connection.uuid = uuid - connection.privilege_level = priv_level - logger.debug(CONNECT_ALLOWED_TMPL % ( - user, host, prog_name, priv_level, uuid)) - return (1, 0) - - def createAuthToken(self, authid, challenge, peeraddr, URI, daemon): - """Return a secure auth token based on the server challenge string. - - Argument authid is what's returned by mungeIdent(). - - """ - hash_ = self._get_hash() - return ":".join( - list(authid[:4]) + - [hmac.new(challenge, authid[4], hash_).digest()] - ) - - def mungeIdent(self, ident): - """Receive (uuid, passphrase) from client. Encrypt the passphrase. - - Also pass client identification info to server for logging: - (user, host, prog name). - - """ - hash_ = self._get_hash() - uuid, passphrase = ident - prog_name = os.path.basename(sys.argv[0]) - if passphrase is None: - passphrase = NO_PASSPHRASE - return (USER, get_hostname(), str(uuid), prog_name, - hash_(passphrase).digest()) - - def _compare_hmacs(self, hmac1, hmac2): - """Compare hmacs as securely as possible.""" - try: - return hmac.compare_hmacs(hmac1, hmac2) - except AttributeError: - # < Python 2.7.7. - return (hmac1 == hmac2) - - def _get_default_hash_name(self): - if hasattr(self, "_default_hash_name"): - return self._default_hash_name - return GLOBAL_CFG.get()['authentication']['hashes'][0] - - def _get_hash(self, hash_name=None): - try: - return self.HASHES[hash_name] - except KeyError: - pass - - hash_name_dest = hash_name - if hash_name is None: - hash_name = self._get_default_hash_name() - - self.HASHES[hash_name_dest] = getattr(hashlib, hash_name) - return self.HASHES[hash_name_dest] - - def _get_hash_name_from_digest_length(self, digest): - if len(digest) in self.LENGTH_HASH_DIGESTS: - return self.LENGTH_HASH_DIGESTS[len(digest)] - for hash_name in OK_HASHES: - hash_ = self._get_hash(hash_name) - len_hash = len(hash_("foo").digest()) - self.LENGTH_HASH_DIGESTS[len_hash] = hash_name - if len_hash == len(digest): - return hash_name - - def _get_no_passphrase_hash(self, hash_name=None): - try: - return self.NO_PASSPHRASE_HASHES[hash_name] - except KeyError: - hash_ = self._get_hash(hash_name) - self.NO_PASSPHRASE_HASHES[hash_name] = ( - hash_(NO_PASSPHRASE).hexdigest()) - return self.NO_PASSPHRASE_HASHES[hash_name] diff --git a/lib/cylc/network/daemon.py b/lib/cylc/network/daemon.py new file mode 100644 index 00000000000..8e9f1845d24 --- /dev/null +++ b/lib/cylc/network/daemon.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2016 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Wrap communications daemon for a suite.""" + +from cylc.network.method import METHOD + +if METHOD == "https": + from cylc.network.https.daemon import CommsDaemon diff --git a/lib/cylc/network/ext_trigger.py b/lib/cylc/network/ext_trigger.py index 36bca2865fa..562e26519ac 100644 --- a/lib/cylc/network/ext_trigger.py +++ b/lib/cylc/network/ext_trigger.py @@ -15,146 +15,11 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +"""Wrap communications daemon for a suite.""" -import sys -from time import sleep -from Queue import Queue, Empty -import Pyro.errors -import cylc.flags -from cylc.network import PYRO_EXT_TRIG_OBJ_NAME -from cylc.network.pyro_base import PyroClient, PyroServer -from cylc.network.suite_broadcast import BroadcastServer -from cylc.network import check_access_priv -from cylc.task_id import TaskID +from cylc.network.method import METHOD -class ExtTriggerServer(PyroServer): - """Server-side external trigger interface.""" - - _INSTANCE = None - - @classmethod - def get_inst(cls): - """Return a singleton instance.""" - if cls._INSTANCE is None: - cls._INSTANCE = cls() - return cls._INSTANCE - - def __init__(self): - super(ExtTriggerServer, self).__init__() - self.queue = Queue() - - def put(self, event_message, event_id): - """Server-side external event trigger interface.""" - - check_access_priv(self, 'full-control') - self.report("ext_trigger") - self.queue.put((event_message, event_id)) - return (True, 'event queued') - - def retrieve(self, itask): - """Match external triggers for a waiting task proxy.""" - - # Note this has to allow multiple same-message triggers to be queued - # and only used one at a time. - - if self.queue.empty(): - return - if len(itask.state.external_triggers) == 0: - return - bcast = BroadcastServer.get_inst() - queued = [] - while True: - try: - queued.append(self.queue.get_nowait()) - except Empty: - break - used = [] - for trig, satisfied in itask.state.external_triggers.items(): - if satisfied: - continue - for qmsg, qid in queued: - if trig == qmsg: - # Matched. - name, point_string = TaskID.split(itask.identity) - # Set trigger satisfied. - itask.state.external_triggers[trig] = True - cylc.flags.pflag = True - # Broadcast the event ID to the cycle point. - if qid is not None: - bcast.put( - [point_string], - ["root"], - [{ - 'environment': { - 'CYLC_EXT_TRIGGER_ID': qid - } - }] - ) - used.append((qmsg, qid)) - break - for q in queued: - if q not in used: - self.queue.put(q) - - -class ExtTriggerClient(PyroClient): - """Client-side external trigger interface.""" - - target_server_object = PYRO_EXT_TRIG_OBJ_NAME - - MAX_N_TRIES = 5 - RETRY_INTVL_SECS = 10.0 - - MSG_SEND_FAILED = "Send message: try %s of %s failed" - MSG_SEND_RETRY = "Retrying in %s seconds, timeout is %s" - MSG_SEND_SUCCEED = "Send message: try %s of %s succeeded" - - def put(self, *args): - return self.call_server_func("put", *args) - - def send_retry(self, event_message, event_id, - max_n_tries, retry_intvl_secs): - """CLI external trigger interface.""" - - max_n_tries = int(max_n_tries or self.__class__.MAX_N_TRIES) - retry_intvl_secs = float( - retry_intvl_secs or self.__class__.RETRY_INTVL_SECS) - - sent = False - i_try = 0 - while not sent and i_try < max_n_tries: - i_try += 1 - try: - self.put(event_message, event_id) - except Pyro.errors.NamingError as exc: - print >> sys.stderr, exc - print self.__class__.MSG_SEND_FAILED % ( - i_try, - max_n_tries, - ) - break - except Exception as exc: - print >> sys.stderr, exc - print self.__class__.MSG_SEND_FAILED % ( - i_try, - max_n_tries, - ) - if i_try >= max_n_tries: - break - print self.__class__.MSG_SEND_RETRY % ( - retry_intvl_secs, - self.pyro_timeout - ) - sleep(retry_intvl_secs) - else: - if i_try > 1: - print self.__class__.MSG_SEND_SUCCEEDED % ( - i_try, - max_n_tries - ) - sent = True - break - if not sent: - sys.exit('ERROR: send failed') - return sent +if METHOD == "https": + from cylc.network.https.ext_trigger import ( + ExtTriggerServer, ExtTriggerClient) diff --git a/lib/cylc/network/https/__init__.py b/lib/cylc/network/https/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/cylc/network/https/base.py b/lib/cylc/network/https/base.py new file mode 100644 index 00000000000..a97f139cd06 --- /dev/null +++ b/lib/cylc/network/https/base.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2016 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Base classes for Pyro servers and clients.""" + +import os +import shlex +from subprocess import Popen, PIPE +import sys +import traceback +import urllib +from uuid import uuid4 +import warnings + +# Ignore incorrect SSL certificate warning from urllib3 via requests. +warnings.filterwarnings("ignore", "Certificate has no `subjectAltName`") + +from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.exceptions import PortFileError +import cylc.flags +from cylc.network import ConnectionDeniedError, NO_PASSPHRASE, handle_proxies +from cylc.network.https.client_reporter import CommsClientReporter +from cylc.owner import is_remote_user, USER +from cylc.registration import RegistrationDB +from cylc.suite_host import get_hostname, is_remote_host +from cylc.suite_env import CylcSuiteEnv, CylcSuiteEnvLoadError +from cylc.version import CYLC_VERSION + + +class BaseCommsServer(object): + """Base class for server-side suite object interfaces.""" + + def __init__(self): + self.client_reporter = CommsClientReporter.get_inst() + + def signout(self): + """Wrap client_reporter.signout.""" + self.client_reporter.signout(self) + + def report(self, command): + """Wrap client_reporter.report.""" + self.client_reporter.report(command, self) + + +class BaseCommsClient(object): + """Base class for client-side suite object interfaces.""" + + ACCESS_DESCRIPTION = 'private' + + def __init__(self, suite, owner=USER, host=None, timeout=None, + port=None, db=None, my_uuid=None, print_uuid=False): + self.suite = suite + self.host = host + self.owner = owner + if timeout is not None: + timeout = float(timeout) + self.timeout = timeout + self.port = port + self.my_uuid = my_uuid or uuid4() + if print_uuid: + print >> sys.stderr, '%s' % self.my_uuid + self.reg_db = RegistrationDB(db) + self.prog_name = os.path.basename(sys.argv[0]) + + def call_server_func(self, category, fname, **fargs): + """Call server_object.fname(*fargs, **fargs).""" + if self.host is None or self.port is None: + self._load_contact_info() + handle_proxies() + payload = fargs.pop("payload", None) + host = self.host + if not self.host.split(".")[0].isdigit(): + host = self.host.split(".")[0] + if host == "localhost": + host = get_hostname().split(".")[0] + url = 'https://%s:%s/%s/%s' % ( + host, self.port, category, fname + ) + if fargs: + params = urllib.urlencode(fargs, doseq=True) + url += "?" + params + return self.get_data_from_url(url, payload) + + def get_data_from_url(self, url, json_data): + requests_ok = True + try: + import requests + except ImportError: + requests_ok = False + else: + if int(requests.__version__.split(".")[0]) < 2: + requests_ok = False + if requests_ok: + return self.get_data_from_url_with_requests(url, json_data) + return self.get_data_from_url_with_urllib2(url, json_data) + + def get_data_from_url_with_requests(self, url, json_data): + import requests + username, password = self._get_auth() + server_cert = self._get_verify() + auth = requests.auth.HTTPDigestAuth(username, password) + if not hasattr(self, "session"): + self.session = requests.Session() + ret = self.session.post( + url, + json=json_data, + verify=self._get_verify(), + proxies={}, + headers=self._get_headers(), + auth=auth + ) + + if ret.status_code == 401: + raise ConnectionDeniedError(url, self.prog_name, + self.ACCESS_DESCRIPTION) + if ret.status_code >= 400: + from cylc.network.https.util import get_exception_from_html + exception_text = get_exception_from_html(ret.text) + if exception_text: + sys.stderr.write(exception_text) + else: + sys.stderr.write(ret.text) + ret.raise_for_status() + try: + return ret.json() + except ValueError: + return ret.text + + def get_data_from_url_with_urllib2(self, url, json_data): + import json + import urllib2 + username, password = self._get_auth() + auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() + auth_manager.add_password(None, url, username, password) + auth = urllib2.HTTPDigestAuthHandler(auth_manager) + opener = urllib2.build_opener(auth, urllib2.HTTPSHandler()) + headers_list = self._get_headers().items() + if json_data: + json_data = json.dumps(json_data) + headers_list.append(('Accept', 'application/json')) + json_headers = {'Content-Type': 'application/json', + 'Content-Length': len(json_data)} + else: + json_data = None + json_headers = {} + opener.addheaders = headers_list + req = urllib2.Request(url, json_data, json_headers) + response = opener.open(req) + + if response.getcode() == 401: + raise ConnectionDeniedError(url, self.prog_name, + self.ACCESS_DESCRIPTION) + response_text = response.read() + if response.getcode() >= 400: + from cylc.network.https.util import get_exception_from_html + exception_text = get_exception_from_html(response_text) + if exception_text: + sys.stderr.write(exception_text) + else: + sys.stderr.write(response_text) + raise Exception("%s HTTP return code" % response.getcode()) + try: + return json.loads(response_text) + except ValueError: + return response_text + +# urllib2.install_opener(opener) +# request = urllib2.Request(url, data=json.dumps(json_data), +# headers=other_headers) +# if sys.version_info < (2, 7, 9): +# return urllib2.urlopen(request) +# return urllib2.urlopen(request, cafile=(server_cert or None)) + + def _get_auth(self): + """Return a user/password Digest Auth.""" + self.pphrase = self.reg_db.load_passphrase( + self.suite, self.owner, self.host) + self.reg_db.cache_passphrase( + self.suite, self.owner, self.host, self.pphrase) + if self.pphrase is None: + return 'anon', NO_PASSPHRASE + return 'cylc', self.pphrase + + def _get_headers(self): + """Return HTTP headers identifying the client.""" + user_agent_string = ( + "cylc/%s prog_name/%s uuid/%s" % ( + CYLC_VERSION, self.prog_name, self.my_uuid + ) + ) + auth_info = "%s@%s" % (USER, get_hostname()) + return {"User-Agent": user_agent_string, + "From": auth_info} + + def _get_verify(self): + """Return the server certificate if possible.""" + if not hasattr(self, "server_cert"): + self.server_cert = self.reg_db.load_item( + self.suite, self.owner, self.host, "certificate") + return self.server_cert + + def _load_contact_info(self): + """Obtain URL info. + + Determine host and port using content in port file, unless already + specified. + + """ + if ((self.host is None or self.port is None) and + 'CYLC_SUITE_RUN_DIR' in os.environ): + # Looks like we are in a running task job, so we should be able to + # use "cylc-suite-env" file under the suite running directory + try: + suite_env = CylcSuiteEnv.load( + self.suite, os.environ['CYLC_SUITE_RUN_DIR']) + except CylcSuiteEnvLoadError: + if cylc.flags.debug: + traceback.print_exc() + else: + self.host = suite_env.suite_host + self.port = suite_env.suite_port + self.owner = suite_env.suite_owner + + if self.host is None or self.port is None: + port_file_path = os.path.join( + GLOBAL_CFG.get( + ['communication', 'ports directory']), self.suite) + if is_remote_host(self.host) or is_remote_user(self.owner): + ssh_tmpl = str(GLOBAL_CFG.get_host_item( + 'remote shell template', self.host, self.owner)) + ssh_tmpl = ssh_tmpl.replace(' %s', '') + user_at_host = '' + if self.owner: + user_at_host = self.owner + '@' + if self.host: + user_at_host += self.host + else: + user_at_host += 'localhost' + r_port_file_path = port_file_path.replace( + os.environ['HOME'], '$HOME') + command = shlex.split(ssh_tmpl) + [ + user_at_host, 'cat', r_port_file_path] + proc = Popen(command, stdout=PIPE, stderr=PIPE) + out, err = proc.communicate() + ret_code = proc.wait() + if ret_code: + if cylc.flags.debug: + print >> sys.stderr, { + "code": ret_code, + "command": command, + "stdout": out, + "stderr": err} + raise PortFileError( + "Port file '%s:%s' not found - suite not running?." % + (user_at_host, r_port_file_path)) + else: + try: + out = open(port_file_path).read() + except IOError: + raise PortFileError( + "Port file '%s' not found - suite not running?." % + (port_file_path)) + lines = out.splitlines() + try: + if self.port is None: + self.port = int(lines[0]) + except (IndexError, ValueError): + raise PortFileError( + "ERROR, bad content in port file: %s" % port_file_path) + if self.host is None: + if len(lines) >= 2: + self.host = lines[1].strip() + else: + self.host = get_hostname() + + def reset(self, *args, **kwargs): + pass + + def signout(self, *args, **kwargs): + pass + + +class BaseCommsClientAnon(BaseCommsClient): + + """Anonymous access class for clients.""" + + ACCESS_DESCRIPTION = 'public' + + def __init__(self, *args, **kwargs): + # We don't necessarily have certificate access for anon suites. + warnings.filterwarnings("ignore", "Unverified HTTPS request") + super(BaseCommsClientAnon, self).__init__(*args, **kwargs) + + def _get_auth(self): + """Return a user/password Digest Auth.""" + return 'anon', NO_PASSPHRASE + + def _get_verify(self): + """Other suites' certificates may not be accessible.""" + return False diff --git a/lib/cylc/network/client_reporter.py b/lib/cylc/network/https/client_reporter.py similarity index 77% rename from lib/cylc/network/client_reporter.py rename to lib/cylc/network/https/client_reporter.py index ac128a205d3..da116cae021 100644 --- a/lib/cylc/network/client_reporter.py +++ b/lib/cylc/network/https/client_reporter.py @@ -22,8 +22,10 @@ import time import cylc.flags +from cylc.network import get_client_info, get_client_connection_denied -class PyroClientReporter(object): + +class CommsClientReporter(object): """For logging cylc client requests with identifying information.""" _INSTANCE = None @@ -34,6 +36,8 @@ class PyroClientReporter(object): LOG_IDENTIFY_TMPL = '[client-identify] %d id requests in PT%dS' LOG_SIGNOUT_TMPL = '[client-sign-out] %s@%s:%s %s' LOG_FORGET_TMPL = '[client-forget] %s' + LOG_CONNECT_DENIED_TMPL = "[client-connect] DENIED %s@%s:%s %s" + LOG_CONNECT_ALLOWED_TMPL = "[client-connect] %s@%s:%s privilege='%s' %s" @classmethod def get_inst(cls): @@ -57,24 +61,27 @@ def report(self, request, server_obj): if threading.current_thread().__class__.__name__ == '_MainThread': # Server methods may be called internally as well as by clients. return + auth_user, prog_name, user, host, uuid, priv_level = get_client_info() name = server_obj.__class__.__name__ - caller = server_obj.getLocalStorage().caller log_me = ( cylc.flags.debug or name in ["SuiteCommandServer", "ExtTriggerServer", "BroadcastServer"] or (name not in ["SuiteIdServer", "TaskMessageServer"] and - caller.uuid not in self.clients)) + uuid not in self.clients)) if log_me: + logging.getLogger("main").debug( + self.__class__.LOG_CONNECT_ALLOWED_TMPL % ( + user, host, prog_name, priv_level, uuid) + ) logging.getLogger("main").info( self.__class__.LOG_COMMAND_TMPL % ( - request, caller.user, caller.host, caller.prog_name, - caller.uuid)) + request, user, host, prog_name, uuid)) if name == "SuiteIdServer": self._num_id_requests += 1 self.report_id_requests() - self.clients[caller.uuid] = datetime.datetime.utcnow() + self.clients[uuid] = datetime.datetime.utcnow() self._housekeep() def report_id_requests(self): @@ -96,6 +103,24 @@ def report_id_requests(self): self._id_start_time = current_time self._num_id_requests = 0 + def report_connection_if_denied(self): + """Log an (un?)successful connection attempt.""" + try: + (auth_user, prog_name, user, host, uuid, + priv_level) = get_client_info() + except Exception: + logging.getLogger("main").warn( + self.__class__.LOG_CONNECT_DENIED_TMPL % ( + "unknown", "unknown", "unknown", "unknown") + ) + return + connection_denied = get_client_connection_denied() + if connection_denied: + logging.getLogger("main").warn( + self.__class__.LOG_CONNECT_DENIED_TMPL % ( + user, host, prog_name, uuid) + ) + def signout(self, server_obj): """Force forget this client (for use by GUI etc.).""" diff --git a/lib/cylc/network/https/daemon.py b/lib/cylc/network/https/daemon.py new file mode 100644 index 00000000000..09210813dc4 --- /dev/null +++ b/lib/cylc/network/https/daemon.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2016 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Wrap HTTPS daemon for a suite.""" + +import binascii +import os +import socket +import sys +import traceback + +from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.network import NO_PASSPHRASE +from cylc.network.https.client_reporter import CommsClientReporter +from cylc.owner import USER +from cylc.registration import RegistrationDB +from cylc.suite_host import get_hostname + +import cherrypy + + +class CommsDaemon(object): + """Wrap HTTPS daemon for a suite.""" + + def __init__(self, suite, suite_dir): + # Suite only needed for back-compat with old clients (see below): + self.suite = suite + + # Figure out the ports we are allowed to use. + base_port = GLOBAL_CFG.get( + ['communication', 'base port']) + max_ports = GLOBAL_CFG.get( + ['communication', 'maximum number of ports']) + self.ok_ports = range( + int(base_port), + int(base_port) + int(max_ports) + ) + + self.reg_db = RegistrationDB() + self.cert = self.reg_db.load_item( + suite, USER, None, "certificate", create_ok=True) + self.pkey = self.reg_db.load_item( + suite, USER, None, "private_key", create_ok=True) + self.suite = suite + passphrase = self.reg_db.load_passphrase(suite, USER, None) + userpassdict = {'cylc': passphrase, 'anon': NO_PASSPHRASE} + get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(userpassdict) + self.get_ha1 = get_ha1 + del passphrase + del userpassdict + self.engine_stopper = lambda: False + self.client_reporter = CommsClientReporter.get_inst() + self.start() + + def start(self): + _ws_init(self) + + def shutdown(self): + """Shutdown the daemon.""" + self.engine_stopper() + + def connect(self, obj, name): + """Connect obj and name to the daemon.""" + import cherrypy + cherrypy.tree.mount(obj, "/" + name) + + def disconnect(self, obj): + """Disconnect obj from the daemon.""" + pass + + def get_port(self): + """Return the daemon port.""" + return self.port + + def report_connection_if_denied(self): + self.client_reporter.report_connection_if_denied() + + +def _ws_init(service_inst, *args, **kwargs): + """Start quick web service.""" + # cherrypy.config["tools.encode.on"] = True + # cherrypy.config["tools.encode.encoding"] = "utf-8" + cherrypy.config["server.socket_host"] = '0.0.0.0' + cherrypy.config["engine.autoreload_on"] = False + cherrypy.config['server.ssl_module'] = 'pyopenSSL' + cherrypy.config['server.ssl_certificate'] = service_inst.cert + cherrypy.config['server.ssl_private_key'] = service_inst.pkey + cherrypy.config['log.screen'] = None + key = binascii.hexlify(os.urandom(16)) + cherrypy.config.update({ + 'tools.auth_digest.on': True, + 'tools.auth_digest.realm': service_inst.suite, + 'tools.auth_digest.get_ha1': service_inst.get_ha1, + 'tools.auth_digest.key': key + }) + cherrypy.tools.connect_log = cherrypy.Tool( + 'on_end_resource', service_inst.report_connection_if_denied) + cherrypy.config['tools.connect_log.on'] = True + host = get_hostname() + service_inst.engine_stopper = lambda: cherrypy.engine.exit() + for port in service_inst.ok_ports: + my_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + my_socket.bind((host, port)) + except socket.error: + # Host busy. + my_socket.close() + continue + my_socket.close() + cherrypy.config["server.socket_port"] = port + try: + cherrypy.engine.start() + except socket.error: + pass + else: + service_inst.port = port + return + raise Exception("No available ports") diff --git a/lib/cylc/network/https/ext_trigger.py b/lib/cylc/network/https/ext_trigger.py new file mode 100644 index 00000000000..e16b7807508 --- /dev/null +++ b/lib/cylc/network/https/ext_trigger.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2016 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cherrypy + +import sys +from time import sleep +from Queue import Queue, Empty +import cylc.flags +from cylc.network import PYRO_EXT_TRIG_OBJ_NAME +from cylc.network.https.base import BaseCommsClient, BaseCommsServer +from cylc.network.https.suite_broadcast import BroadcastServer +from cylc.network import check_access_priv +from cylc.task_id import TaskID + + +class ExtTriggerServer(BaseCommsServer): + """Server-side external trigger interface.""" + + _INSTANCE = None + + @classmethod + def get_inst(cls): + """Return a singleton instance.""" + if cls._INSTANCE is None: + cls._INSTANCE = cls() + return cls._INSTANCE + + def __init__(self): + super(ExtTriggerServer, self).__init__() + self.queue = Queue() + + @cherrypy.expose + @cherrypy.tools.json_out() + def put(self, event_message, event_id): + """Server-side external event trigger interface.""" + + check_access_priv(self, 'full-control') + self.report("ext_trigger") + self.queue.put((event_message, event_id)) + return (True, 'event queued') + + def retrieve(self, itask): + """Match external triggers for a waiting task proxy.""" + + # Note this has to allow multiple same-message triggers to be queued + # and only used one at a time. + + if self.queue.empty(): + return + if len(itask.state.external_triggers) == 0: + return + bcast = BroadcastServer.get_inst() + queued = [] + while True: + try: + queued.append(self.queue.get_nowait()) + except Empty: + break + used = [] + for trig, satisfied in itask.state.external_triggers.items(): + if satisfied: + continue + for qmsg, qid in queued: + if trig == qmsg: + # Matched. + name, point_string = TaskID.split(itask.identity) + # Set trigger satisfied. + itask.state.external_triggers[trig] = True + cylc.flags.pflag = True + # Broadcast the event ID to the cycle point. + if qid is not None: + bcast.put( + [point_string], + ["root"], + [{ + 'environment': { + 'CYLC_EXT_TRIGGER_ID': qid + } + }], + not_from_client=True + ) + used.append((qmsg, qid)) + break + for q in queued: + if q not in used: + self.queue.put(q) + + +class ExtTriggerClient(BaseCommsClient): + """Client-side external trigger interface.""" + + MAX_N_TRIES = 5 + RETRY_INTVL_SECS = 10.0 + + MSG_SEND_FAILED = "Send message: try %s of %s failed" + MSG_SEND_RETRY = "Retrying in %s seconds, timeout is %s" + MSG_SEND_SUCCEED = "Send message: try %s of %s succeeded" + + def put(self, event_message, event_id): + return self.call_server_func("ext-trigger", "put", + event_message=event_message, + event_id=event_id) + + def send_retry(self, event_message, event_id, + max_n_tries, retry_intvl_secs): + """CLI external trigger interface.""" + + max_n_tries = int(max_n_tries or self.__class__.MAX_N_TRIES) + retry_intvl_secs = float( + retry_intvl_secs or self.__class__.RETRY_INTVL_SECS) + + sent = False + i_try = 0 + while not sent and i_try < max_n_tries: + i_try += 1 + try: + self.put(event_message, event_id) + except Exception as exc: + print >> sys.stderr, exc + print self.__class__.MSG_SEND_FAILED % ( + i_try, + max_n_tries, + ) + if i_try >= max_n_tries: + break + print self.__class__.MSG_SEND_RETRY % ( + retry_intvl_secs, + self.timeout + ) + sleep(retry_intvl_secs) + else: + if i_try > 1: + print self.__class__.MSG_SEND_SUCCEEDED % ( + i_try, + max_n_tries + ) + sent = True + break + if not sent: + sys.exit('ERROR: send failed') + return sent diff --git a/lib/cylc/network/https/port_scan.py b/lib/cylc/network/https/port_scan.py new file mode 100644 index 00000000000..0b292e49a33 --- /dev/null +++ b/lib/cylc/network/https/port_scan.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2016 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Port scan utilities.""" + +from multiprocessing import cpu_count, Pool +import sys +from time import sleep +import traceback +from uuid import uuid4 + +from cylc.cfgspec.globalcfg import GLOBAL_CFG +import cylc.flags +from cylc.network import PYRO_SUITEID_OBJ_NAME, NO_PASSPHRASE +from cylc.network.https.suite_state import SuiteStillInitialisingError +from cylc.network.https.suite_identifier import ( + SuiteIdClientAnon, SuiteIdClient) +from cylc.owner import USER +from cylc.registration import RegistrationDB +from cylc.suite_host import get_hostname, is_remote_host + +import requests + + +def scan(host=None, db=None, timeout=None): + """Scan ports, return a list of suites found: [(port, suite.identify())]. + + Note that we could easily scan for a given suite+owner and return its + port instead of reading port files, but this may not always be fast enough. + """ + if host is None: + host = get_hostname() + base_port = GLOBAL_CFG.get( + ['communication', 'base port']) + last_port = base_port + GLOBAL_CFG.get( + ['communication', 'maximum number of ports']) + if timeout: + timeout = float(timeout) + else: + timeout = None + + reg_db = RegistrationDB(db) + results = [] + my_uuid = uuid4() + for port in range(base_port, last_port): + client = SuiteIdClientAnon(None, host=host, port=port, my_uuid=my_uuid) + try: + result = (port, client.identify()) + except requests.exceptions.RequestException as exc: + if cylc.flags.debug: + traceback.print_exc() + continue + except Exception as exc: + if cylc.flags.debug: + traceback.print_exc() + raise + else: + owner = result[1].get('owner') + name = result[1].get('name') + states = result[1].get('states', None) + if cylc.flags.debug: + print ' suite:', name, owner + if states is None: + # This suite keeps its state info private. + # Try again with the passphrase if I have it. + pphrase = reg_db.load_passphrase(name, owner, host) + if pphrase: + client = SuiteIdClient(name, owner=owner, host=host, + port=port, my_uuid=my_uuid, + timeout=timeout) + try: + result = (port, client.identify()) + except Exception: + # Nope (private suite, wrong passphrase). + if cylc.flags.debug: + print ' (wrong passphrase)' + else: + reg_db.cache_passphrase( + name, owner, host, pphrase) + if cylc.flags.debug: + print ' (got states with passphrase)' + results.append(result) + return results + + +def scan_all(hosts=None, reg_db_path=None, timeout=None): + """Scan all hosts.""" + if not hosts: + hosts = GLOBAL_CFG.get(["suite host scanning", "hosts"]) + # Ensure that it does "localhost" only once + hosts = set(hosts) + for host in list(hosts): + if not is_remote_host(host): + hosts.remove(host) + hosts.add("localhost") + proc_pool_size = GLOBAL_CFG.get(["process pool size"]) + if proc_pool_size is None: + proc_pool_size = cpu_count() + if proc_pool_size > len(hosts): + proc_pool_size = len(hosts) + proc_pool = Pool(proc_pool_size) + async_results = {} + for host in hosts: + async_results[host] = proc_pool.apply_async( + scan, [host, reg_db_path, timeout]) + proc_pool.close() + scan_results = [] + scan_results_hosts = [] + while async_results: + sleep(0.05) + for host, async_result in async_results.items(): + if async_result.ready(): + async_results.pop(host) + try: + res = async_result.get() + except Exception: + if cylc.flags.debug: + traceback.print_exc() + else: + scan_results.extend(res) + scan_results_hosts.extend([host] * len(res)) + proc_pool.join() + return zip(scan_results_hosts, scan_results) diff --git a/lib/cylc/network/https/suite_broadcast.py b/lib/cylc/network/https/suite_broadcast.py new file mode 100644 index 00000000000..bc44e91ee44 --- /dev/null +++ b/lib/cylc/network/https/suite_broadcast.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2016 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import json +import sys +import logging +import cPickle as pickle +import threading + +from cylc.broadcast_report import ( + get_broadcast_change_iter, + get_broadcast_change_report, + get_broadcast_bad_options_report) +from cylc.cycling.loader import get_point, standardise_point_string +from cylc.wallclock import get_current_time_string +from cylc.network import PYRO_BCAST_OBJ_NAME +from cylc.network.https.base import BaseCommsClient, BaseCommsServer +from cylc.network.https.util import unicode_encode +from cylc.network import check_access_priv +from cylc.task_id import TaskID +from cylc.rundb import CylcSuiteDAO + +import cherrypy + + +class BroadcastServer(BaseCommsServer): + """Server-side suite broadcast interface. + + Examples: + self.settings['*']['root'] = {'environment': {'FOO': 'bar'}} + self.settings['20100808T06Z']['root'] = {'command scripting': 'stuff'} + """ + + _INSTANCE = None + ALL_CYCLE_POINTS_STRS = ["*", "all-cycle-points", "all-cycles"] + TABLE_BROADCAST_EVENTS = CylcSuiteDAO.TABLE_BROADCAST_EVENTS + TABLE_BROADCAST_STATES = CylcSuiteDAO.TABLE_BROADCAST_STATES + + @classmethod + def get_inst(cls, linearized_ancestors=None): + """Return a singleton instance. + + On 1st call, instantiate the singleton. + Argument linearized_ancestors is only relevant on 1st call. + + """ + if cls._INSTANCE is None: + cls._INSTANCE = cls(linearized_ancestors) + return cls._INSTANCE + + def __init__(self, linearized_ancestors): + super(BroadcastServer, self).__init__() + self.log = logging.getLogger('main') + self.settings = {} + self.db_inserts_map = { + self.TABLE_BROADCAST_EVENTS: [], + self.TABLE_BROADCAST_STATES: []} + self.db_deletes_map = { + self.TABLE_BROADCAST_STATES: []} + self.linearized_ancestors = linearized_ancestors + self.lock = threading.RLock() + + def _prune(self): + """Remove empty leaves left by unsetting broadcast values. + + Return a list of pruned settings in the form: + + [ + ["20200202", "foo", "command scripting"], + ["20020202", "bar", "environment", "BAR"], + ] + """ + with self.lock: + prunes = [] + stuff_stack = [([], self.settings, True)] + while stuff_stack: + keys, stuff, is_new = stuff_stack.pop() + if is_new: + stuff_stack.append((keys, stuff, False)) + for key, value in stuff.items(): + if isinstance(value, dict): + stuff_stack.append((keys + [key], value, True)) + else: + for key, value in stuff.items(): + if value in [None, {}]: + del stuff[key] + prunes.append(keys + [key]) + return prunes + + def _addict(self, target, source): + """Recursively add source dict to target dict.""" + for key, val in source.items(): + if isinstance(val, dict): + if key not in target: + target[key] = {} + self._addict(target[key], val) + else: + target[key] = source[key] + + @cherrypy.expose + @cherrypy.tools.json_in() + @cherrypy.tools.json_out() + def put(self, point_strings=None, namespaces=None, settings=None, + not_from_client=False): + """Add new broadcast settings (server side interface). + + Return a tuple (modified_settings, bad_options) where: + modified_settings is list of modified settings in the form: + [("20200202", "foo", {"command scripting": "true"}, ...] + bad_options is as described in the docstring for self.clear(). + """ + check_access_priv(self, 'full-control') + self.report('broadcast_put') + if not not_from_client: + point_strings = ( + cherrypy.request.json.get("point_strings", point_strings)) + namespaces = ( + cherrypy.request.json.get("namespaces", namespaces)) + settings = ( + cherrypy.request.json.get("settings", settings)) + point_strings = unicode_encode(point_strings) + namespaces = unicode_encode(namespaces) + settings = unicode_encode(settings) + + modified_settings = [] + bad_point_strings = [] + bad_namespaces = [] + + with self.lock: + for setting in settings: + for point_string in point_strings: + # Standardise the point and check its validity. + bad_point = False + try: + point_string = standardise_point_string(point_string) + except Exception as exc: + if point_string != '*': + bad_point_strings.append(point_string) + bad_point = True + if not bad_point and point_string not in self.settings: + self.settings[point_string] = {} + for namespace in namespaces: + if namespace not in self.linearized_ancestors: + bad_namespaces.append(namespace) + elif not bad_point: + if namespace not in self.settings[point_string]: + self.settings[point_string][namespace] = {} + self._addict( + self.settings[point_string][namespace], + setting) + modified_settings.append( + (point_string, namespace, setting)) + + # Log the broadcast + self._append_db_queue(modified_settings) + self.log.info(get_broadcast_change_report(modified_settings)) + + bad_options = {} + if bad_point_strings: + bad_options["point_strings"] = bad_point_strings + if bad_namespaces: + bad_options["namespaces"] = bad_namespaces + return modified_settings, bad_options + + @cherrypy.expose + @cherrypy.tools.json_out() + def get(self, task_id=None): + """Retrieve all broadcast variables that target a given task ID.""" + check_access_priv(self, 'full-read') + self.report('broadcast_get') + if task_id == "None": + task_id = None + if not task_id: + # all broadcast settings requested + return self.settings + try: + name, point_string = TaskID.split(task_id) + except ValueError: + raise Exception("Can't split task_id %s" % task_id) + + ret = {} + # The order is: + # all:root -> all:FAM -> ... -> all:task + # -> tag:root -> tag:FAM -> ... -> tag:task + for cycle in self.ALL_CYCLE_POINTS_STRS + [point_string]: + if cycle not in self.settings: + continue + for namespace in reversed(self.linearized_ancestors[name]): + if namespace in self.settings[cycle]: + self._addict(ret, self.settings[cycle][namespace]) + return ret + + @cherrypy.expose + @cherrypy.tools.json_out() + def expire(self, cutoff=None): + """Clear all settings targeting cycle points earlier than cutoff.""" + point_strings = [] + cutoff_point = None + if cutoff is not None: + cutoff_point = get_point(str(cutoff)) + with self.lock: + for point_string in self.settings: + if cutoff_point is None or ( + point_string not in self.ALL_CYCLE_POINTS_STRS and + get_point(point_string) < cutoff_point): + point_strings.append(point_string) + if not point_strings: + return (None, {"expire": [cutoff]}) + return self.clear(point_strings=point_strings) + + @cherrypy.expose + @cherrypy.tools.json_in() + @cherrypy.tools.json_out() + def clear(self, point_strings=None, namespaces=None, cancel_settings=None): + """Clear settings globally, or for listed namespaces and/or points. + + Return a tuple (modified_settings, bad_options), where: + * modified_settings is similar to the return value of the "put" method, + but for removed settings. + * bad_options is a dict in the form: + {"point_strings": ["20020202", ..."], ...} + The dict is only populated if there are options not associated with + previous broadcasts. The keys can be: + * point_strings: a list of bad point strings. + * namespaces: a list of bad namespaces. + * cancel: a list of tuples. Each tuple contains the keys of a bad + setting. + """ + + if hasattr(cherrypy.request, "json"): + point_strings = ( + cherrypy.request.json.get("point_strings", point_strings)) + namespaces = ( + cherrypy.request.json.get("namespaces", namespaces)) + cancel_settings = ( + cherrypy.request.json.get("cancel_settings", cancel_settings)) + point_strings = unicode_encode(point_strings) + namespaces = unicode_encode(namespaces) + settings = unicode_encode(settings) + # If cancel_settings defined, only clear specific settings + cancel_keys_list = self._settings_to_keys_list(cancel_settings) + + # Clear settings + modified_settings = [] + with self.lock: + for point_string, point_string_settings in self.settings.items(): + if point_strings and point_string not in point_strings: + continue + for namespace, namespace_settings in ( + point_string_settings.items()): + if namespaces and namespace not in namespaces: + continue + stuff_stack = [([], namespace_settings)] + while stuff_stack: + keys, stuff = stuff_stack.pop() + for key, value in stuff.items(): + if isinstance(value, dict): + stuff_stack.append((keys + [key], value)) + elif (not cancel_keys_list or + keys + [key] in cancel_keys_list): + stuff[key] = None + setting = {key: value} + for rkey in reversed(keys): + setting = {rkey: setting} + modified_settings.append( + (point_string, namespace, setting)) + + # Prune any empty branches + bad_options = self._get_bad_options( + self._prune(), point_strings, namespaces, cancel_keys_list) + + # Log the broadcast + self._append_db_queue(modified_settings, is_cancel=True) + self.log.info( + get_broadcast_change_report(modified_settings, is_cancel=True)) + if bad_options: + self.log.error(get_broadcast_bad_options_report(bad_options)) + + return (modified_settings, bad_options) + + @staticmethod + def _settings_to_keys_list(settings): + """Return a list containing each setting dict keys as a list. + + E.g. Each setting in settings may look like: + {"foo": {"bar": {"baz": 1}}} + + An element of the returned list will look like: + ["foo", "bar", "baz"] + + """ + keys_list = [] + if settings: + for setting in settings: + stuff_stack = [([], setting)] + while stuff_stack: + keys, stuff = stuff_stack.pop() + for key, value in stuff.items(): + if isinstance(value, dict): + stuff_stack.append((keys + [key], value)) + else: + keys_list.append(keys + [key]) + return keys_list + + def dump(self, file_): + """Write broadcast variables to the state dump file.""" + with self.lock: + pickle.dump(self.settings, file_) + file_.write("\n") + + def load(self, pickled_settings): + """Load broadcast variables from the state dump file.""" + with self.lock: + self.settings = pickle.loads(pickled_settings) + + # Ensure database table is in sync + modified_settings = [] + for point_string, point_string_settings in self.settings.items(): + for namespace, namespace_settings in ( + point_string_settings.items()): + stuff_stack = [([], namespace_settings)] + while stuff_stack: + keys, stuff = stuff_stack.pop() + for key, value in stuff.items(): + if isinstance(value, dict): + stuff_stack.append((keys + [key], value)) + else: + setting = {key: value} + for rkey in reversed(keys): + setting = {rkey: setting} + modified_settings.append( + (point_string, namespace, setting)) + for broadcast_change in get_broadcast_change_iter(modified_settings): + self.db_inserts_map[self.TABLE_BROADCAST_STATES].append({ + "point": broadcast_change["point"], + "namespace": broadcast_change["namespace"], + "key": broadcast_change["key"], + "value": broadcast_change["value"]}) + + def _get_dump(self): + """Return broadcast variables as written to the state dump file.""" + with self.lock: + return pickle.dumps(self.settings) + "\n" + + @classmethod + def _get_bad_options( + cls, prunes, point_strings, namespaces, cancel_keys_list): + """Return unpruned namespaces and/or point_strings options.""" + cancel_keys_list = [ + tuple(cancel_keys) for cancel_keys in cancel_keys_list] + bad_options = { + "point_strings": None, "namespaces": None, "cancel": None} + # 1. Populate the bad_options dict where applicable. + # 2. Remove keys if they are found in "prunes". + # 3. Remove key in bad_options if it becomes empty. + for opt_name, opt_list, opt_test in [ + ("point_strings", point_strings, cls._point_string_in_prunes), + ("namespaces", namespaces, cls._namespace_in_prunes), + ("cancel", cancel_keys_list, cls._cancel_keys_in_prunes)]: + if opt_list: + bad_options[opt_name] = set(opt_list) + for opt in opt_list: + if opt_test(prunes, opt): + bad_options[opt_name].discard(opt) + if not bad_options[opt_name]: + break + for key, value in bad_options.items(): + if value: + bad_options[key] = list(value) + else: + del bad_options[key] + return bad_options + + @staticmethod + def _point_string_in_prunes(prunes, point_string): + """Is point_string pruned?""" + return point_string in [prune[0] for prune in prunes] + + @staticmethod + def _namespace_in_prunes(prunes, namespace): + """Is namespace pruned?""" + return namespace in [prune[1] for prune in prunes if prune[1:]] + + @staticmethod + def _cancel_keys_in_prunes(prunes, cancel_keys): + """Is cancel_keys pruned?""" + return (list(cancel_keys) in + [prune[2:] for prune in prunes if prune[2:]]) + + def _append_db_queue(self, modified_settings, is_cancel=False): + """Update the queue to the runtime DB.""" + now = get_current_time_string(display_sub_seconds=True) + for broadcast_change in ( + get_broadcast_change_iter(modified_settings, is_cancel)): + broadcast_change["time"] = now + self.db_inserts_map[self.TABLE_BROADCAST_EVENTS].append( + broadcast_change) + if is_cancel: + self.db_deletes_map[self.TABLE_BROADCAST_STATES].append({ + "point": broadcast_change["point"], + "namespace": broadcast_change["namespace"], + "key": broadcast_change["key"]}) + # Delete statements are currently executed before insert + # statements, so we should clear out any insert statements that + # are deleted here. + # (Not the most efficient logic here, but unless we have a + # large number of inserts, then this should not be a big + # concern.) + inserts = [] + for insert in self.db_inserts_map[self.TABLE_BROADCAST_STATES]: + if any([insert[key] != broadcast_change[key] for key in + ["point", "namespace", "key"]]): + inserts.append(insert) + self.db_inserts_map[self.TABLE_BROADCAST_STATES] = inserts + else: + self.db_inserts_map[self.TABLE_BROADCAST_STATES].append({ + "point": broadcast_change["point"], + "namespace": broadcast_change["namespace"], + "key": broadcast_change["key"], + "value": broadcast_change["value"]}) + + +class BroadcastClient(BaseCommsClient): + """Client-side suite broadcast interface.""" + + def broadcast(self, cmd, **kwargs): + if cmd in ["get", "expire"]: + return self.call_server_func("broadcast", cmd, **kwargs) + return self.call_server_func("broadcast", cmd, payload=kwargs) diff --git a/lib/cylc/network/https/suite_command.py b/lib/cylc/network/https/suite_command.py new file mode 100644 index 00000000000..bb7127fa27c --- /dev/null +++ b/lib/cylc/network/https/suite_command.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2016 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import ast +import sys +import os +from Queue import Queue + +import cylc.flags +from cylc.network import PYRO_CMD_OBJ_NAME +from cylc.network.https.base import BaseCommsClient, BaseCommsServer +from cylc.network import check_access_priv + +import cherrypy + + +class SuiteCommandServer(BaseCommsServer): + """Server-side suite command interface.""" + + def __init__(self): + super(SuiteCommandServer, self).__init__() + self.queue = Queue() + + @cherrypy.expose + @cherrypy.tools.json_out() + def set_stop_cleanly(self, kill_active_tasks=False): + kill_active_tasks = ast.literal_eval(kill_active_tasks) + return self._put("set_stop_cleanly", + None, {"kill_active_tasks": kill_active_tasks}) + + @cherrypy.expose + @cherrypy.tools.json_out() + def stop_now(self): + return self._put("stop_now", None) + + @cherrypy.expose + @cherrypy.tools.json_out() + def set_stop_after_point(self, point_string): + return self._put("set_stop_after_point", (point_string,)) + + @cherrypy.expose + @cherrypy.tools.json_out() + def set_stop_after_clock_time(self, datetime_string): + return self._put("set_stop_after_clock_time", (datetime_string,)) + + @cherrypy.expose + @cherrypy.tools.json_out() + def set_stop_after_task(self, task_id): + return self._put("set_stop_after_task", (task_id,)) + + @cherrypy.expose + @cherrypy.tools.json_out() + def release_suite(self): + return self._put("release_suite", None) + + @cherrypy.expose + @cherrypy.tools.json_out() + def release_tasks(self, items): + if not isinstance(items, list): + items = [items] + return self._put("release_tasks", (items,)) + + @cherrypy.expose + @cherrypy.tools.json_out() + def remove_cycle(self, point_string, spawn=False): + spawn = ast.literal_eval(spawn) + return self._put("remove_cycle", (point_string,), {"spawn": spawn}) + + @cherrypy.expose + @cherrypy.tools.json_out() + def remove_tasks(self, items, spawn=False): + spawn = ast.literal_eval(spawn) + if not isinstance(items, list): + items = [items] + return self._put("remove_tasks", (items,), {"spawn": spawn}) + + @cherrypy.expose + @cherrypy.tools.json_out() + def hold_suite(self): + return self._put("hold_suite", None) + + @cherrypy.expose + @cherrypy.tools.json_out() + def hold_after_point_string(self, point_string): + return self._put("hold_after_point_string", (point_string,)) + + @cherrypy.expose + @cherrypy.tools.json_out() + def hold_tasks(self, items): + if not isinstance(items, list): + items = [items] + return self._put("hold_tasks", (items,)) + + @cherrypy.expose + @cherrypy.tools.json_out() + def set_runahead(self, interval=None): + interval = ast.literal_eval(interval) + return self._put("set_runahead", None, {"interval": interval}) + + @cherrypy.expose + @cherrypy.tools.json_out() + def set_verbosity(self, level): + return self._put("set_verbosity", (level,)) + + @cherrypy.expose + @cherrypy.tools.json_out() + def reset_task_states(self, items, state=None): + if not isinstance(items, list): + items = [items] + return self._put("reset_task_states", (items,), {"state": state}) + + @cherrypy.expose + @cherrypy.tools.json_out() + def trigger_tasks(self, items): + if not isinstance(items, list): + items = [items] + items = [str(item) for item in items] + return self._put("trigger_tasks", (items,)) + + @cherrypy.expose + @cherrypy.tools.json_out() + def dry_run_tasks(self, items): + if not isinstance(items, list): + items = [items] + return self._put("dry_run_tasks", (items,)) + + @cherrypy.expose + @cherrypy.tools.json_out() + def nudge(self): + return self._put("nudge", None) + + @cherrypy.expose + @cherrypy.tools.json_out() + def insert_tasks(self, items, stop_point_string=None): + if not isinstance(items, list): + items = [items] + if stop_point_string == "None": + stop_point_string = None + return self._put("insert_tasks", (items,), + {"stop_point_string": stop_point_string}) + + @cherrypy.expose + @cherrypy.tools.json_out() + def reload_suite(self): + return self._put("reload_suite", None) + + @cherrypy.expose + @cherrypy.tools.json_out() + def poll_tasks(self, items=None): + if items is not None and not isinstance(items, list): + items = [items] + return self._put("poll_tasks", (items,)) + + @cherrypy.expose + @cherrypy.tools.json_out() + def kill_tasks(self, items): + if not isinstance(items, list): + items = [items] + return self._put("kill_tasks", (items,)) + + @cherrypy.expose + @cherrypy.tools.json_out() + def spawn_tasks(self, items): + if not isinstance(items, list): + items = [items] + return self._put("kill_tasks", (items,)) + + def _put(self, command, command_args, command_kwargs=None): + if command_args is None: + command_args = tuple() + if command_kwargs is None: + command_kwargs = {} + if 'stop' in command: + check_access_priv(self, 'shutdown') + else: + check_access_priv(self, 'full-control') + self.report(command) + self.queue.put((command, command_args, command_kwargs)) + return (True, 'Command queued') + + def get_queue(self): + return self.queue + + +class SuiteCommandClient(BaseCommsClient): + """Client-side suite command interface.""" + + def put_command(self, command, **arg_dict): + success, msg = self.call_server_func("command", command, + **arg_dict) + return (success, msg) diff --git a/lib/cylc/network/https/suite_identifier.py b/lib/cylc/network/https/suite_identifier.py new file mode 100644 index 00000000000..35bb258792a --- /dev/null +++ b/lib/cylc/network/https/suite_identifier.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2016 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# A minimal server/client pair to allow client programs to identify +# what suite is running at a given cylc port - by suite name and owner. + +import cylc.flags +from cylc.network.https.base import ( + BaseCommsServer, BaseCommsClient, BaseCommsClientAnon) +from cylc.network.https.suite_state import StateSummaryServer +from cylc.network import access_priv_ok +from cylc.config import SuiteConfig + +import cherrypy + + +class SuiteIdServer(BaseCommsServer): + """Server-side identification interface.""" + + _INSTANCE = None + + @classmethod + def get_inst(cls, name=None, owner=None): + """Return a singleton instance.""" + if cls._INSTANCE is None: + cls._INSTANCE = cls(name, owner) + return cls._INSTANCE + + def __init__(self, name, owner): + self.owner = owner + self.name = name + super(SuiteIdServer, self).__init__() + + @cherrypy.expose + @cherrypy.tools.json_out() + def identify(self): + self.report("identify") + result = {} + if access_priv_ok(self, "identity"): + result['name'] = self.name + result['owner'] = self.owner + if access_priv_ok(self, "description"): + config = SuiteConfig.get_inst() + result['title'] = config.cfg['title'] + result['description'] = config.cfg['description'] + if access_priv_ok(self, "state-totals"): + result['states'] = StateSummaryServer.get_inst().get_state_totals() + result['update-time'] = ( + StateSummaryServer.get_inst().get_summary_update_time()) + return result + + +class SuiteIdClient(BaseCommsClient): + """Client-side suite identity nterface.""" + + def identify(self): + return self.call_server_func("id", "identify") + + +class SuiteIdClientAnon(BaseCommsClientAnon): + """Client-side suite identity nterface.""" + + def identify(self): + return self.call_server_func("id", "identify") diff --git a/lib/cylc/network/https/suite_info.py b/lib/cylc/network/https/suite_info.py new file mode 100644 index 00000000000..a1449d1c93b --- /dev/null +++ b/lib/cylc/network/https/suite_info.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2016 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import ast +import sys + +import cylc.flags +from cylc.network import PYRO_INFO_OBJ_NAME +from cylc.network.https.base import ( + BaseCommsClient, BaseCommsClientAnon, BaseCommsServer) +from cylc.network import check_access_priv + +import cherrypy + + +class SuiteInfoServer(BaseCommsServer): + """Server-side suite information interface.""" + + def __init__(self, info_commands): + super(SuiteInfoServer, self).__init__() + self.commands = info_commands + + @cherrypy.expose + @cherrypy.tools.json_out() + def ping_suite(self): + return self._put("ping_suite", None) + + @cherrypy.expose + @cherrypy.tools.json_out() + def get_cylc_version(self): + return self._put("get_cylc_version", None) + + @cherrypy.expose + @cherrypy.tools.json_out() + def ping_task(self, task_id, exists_only=False): + if isinstance(exists_only, basestring): + exists_only = ast.literal_eval(exists_only) + return self._put("ping_task", (task_id,), + {"exists_only": exists_only}) + + @cherrypy.expose + @cherrypy.tools.json_out() + def get_task_jobfile_path(self, task_id): + return self._put("get_task_jobfile_path", (task_id,)) + + @cherrypy.expose + @cherrypy.tools.json_out() + def get_suite_info(self): + return self._put("get_suite_info", None) + + @cherrypy.expose + @cherrypy.tools.json_out() + def get_task_info(self, name): + return self._put("get_task_info", (name,)) + + @cherrypy.expose + @cherrypy.tools.json_out() + def get_all_families(self, exclude_root=False): + if isinstance(exclude_root, basestring): + exclude_root = ast.literal_eval(exclude_root) + return self._put("get_all_families", None, + {"exclude_root": exclude_root}) + + @cherrypy.expose + @cherrypy.tools.json_out() + def get_triggering_families(self): + return self._put("get_triggering_families", None) + + @cherrypy.expose + @cherrypy.tools.json_out() + def get_first_parent_descendants(self): + return self._put("get_first_parent_descendants", None) + + @cherrypy.expose + @cherrypy.tools.json_out() + def get_first_parent_ancestors(self, pruned=None): + if isinstance(pruned, basestring): + pruned = ast.literal_eval(pruned) + return self._put("get_first_parent_ancestors", None, + {"pruned": pruned}) + + @cherrypy.expose + @cherrypy.tools.json_out() + def get_graph_raw(self, start_point_string, stop_point_string, + group_nodes=None, ungroup_nodes=None, + ungroup_recursive=False, group_all=False, + ungroup_all=False): + if isinstance(group_nodes, basestring): + group_nodes = ast.literal_eval(group_nodes) + if isinstance(ungroup_nodes, basestring): + ungroup_nodes = ast.literal_eval(ungroup_nodes) + if isinstance(ungroup_recursive, basestring): + ungroup_recursive = ast.literal_eval(ungroup_recursive) + if isinstance(group_all, basestring): + group_all = ast.literal_eval(group_all) + if isinstance(ungroup_all, basestring): + ungroup_all = ast.literal_eval(ungroup_all) + return self._put( + "get_graph_raw", (start_point_string, stop_point_string), + {"group_nodes": group_nodes, + "ungroup_nodes": ungroup_nodes, + "ungroup_recursive": ungroup_recursive, + "group_all": group_all, + "ungroup_all": ungroup_all} + ) + + @cherrypy.expose + @cherrypy.tools.json_out() + def get_task_requisites(self, name, point_string): + return self._put("get_task_requisites", (name, point_string)) + + def _put(self, command, command_args, command_kwargs=None): + if command_args is None: + command_args = tuple() + if command_kwargs is None: + command_kwargs = {} + if ('ping' in command or 'version' in command): + # Free info. + pass + elif 'suite' in command and 'info' in command: + # Suite title and description only. + check_access_priv(self, 'description') + else: + check_access_priv(self, 'full-read') + self.report(command) + return self.commands[command](*command_args, **command_kwargs) + + +class SuiteInfoClient(BaseCommsClient): + """Client-side suite information interface.""" + + def get_info(self, command, **arg_dict): + return self.call_server_func("info", command, **arg_dict) + + +class SuiteInfoClientAnon(BaseCommsClientAnon): + """Anonymous client-side suite information interface.""" + + def get_info(self, command, **arg_dict): + return self.call_server_func("info", command, **arg_dict) diff --git a/lib/cylc/network/https/suite_log.py b/lib/cylc/network/https/suite_log.py new file mode 100644 index 00000000000..1548b606de1 --- /dev/null +++ b/lib/cylc/network/https/suite_log.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2016 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +from cylc.network.https.base import BaseCommsClient, BaseCommsServer +from cylc.network import check_access_priv + +import cherrypy + + +class SuiteLogServer(BaseCommsServer): + """Server-side suite log interface.""" + + def __init__(self, log): + super(SuiteLogServer, self).__init__() + self.log = log + self.err_file = log.get_err_path() + + def _get_err_has_changed(self, prev_err_size): + """Return True if the file has changed size compared to prev_size.""" + return self._get_err_size() != prev_err_size + + def _get_err_size(self): + """Return the os.path.getsize result for the error file.""" + + try: + size = os.path.getsize(self.err_file) + except (IOError, OSError) as exc: + self.log.warn("Could not read suite err log file: %s" % exc) + return 0 + return size + + @cherrypy.expose + @cherrypy.tools.json_out() + def get_err_content(self, prev_size, max_lines): + """Return the content and new size of the error file.""" + check_access_priv(self, 'full-read') + self.report("get_err_content") + prev_size = int(prev_size) + max_lines = int(max_lines) + if not self._get_err_has_changed(prev_size): + return [], prev_size + try: + f = open(self.err_file, "r") + f.seek(prev_size) + new_content = f.read() + f.close() + size = self._get_err_size() + except (IOError, OSError) as e: + self.log.warning("Could not read suite err log file: %s" % e) + return "", prev_size + new_content_lines = new_content.splitlines()[-max_lines:] + return "\n".join(new_content_lines), size + + +class SuiteLogClient(BaseCommsClient): + """Client-side suite log interface.""" + + def get_err_content(self, prev_size, max_lines): + return self.call_server_func("log", "get_err_content", + prev_size=prev_size, max_lines=max_lines) diff --git a/lib/cylc/network/https/suite_state.py b/lib/cylc/network/https/suite_state.py new file mode 100644 index 00000000000..d415fecbf70 --- /dev/null +++ b/lib/cylc/network/https/suite_state.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2016 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import re +import time +import datetime + +import cylc.flags +from cylc.task_id import TaskID +from cylc.wallclock import TIME_ZONE_LOCAL_INFO, TIME_ZONE_UTC_INFO +from cylc.config import SuiteConfig +from cylc.network import PYRO_STATE_OBJ_NAME +from cylc.network.https.base import BaseCommsClient, BaseCommsServer +from cylc.network import check_access_priv +from cylc.task_state import ( + TASK_STATUS_RUNAHEAD, TASK_STATUS_HELD, TASK_STATUS_WAITING, + TASK_STATUS_EXPIRED, TASK_STATUS_QUEUED, TASK_STATUS_READY, + TASK_STATUS_SUBMITTED, TASK_STATUS_SUBMIT_FAILED, + TASK_STATUS_SUBMIT_RETRYING, TASK_STATUS_RUNNING, TASK_STATUS_SUCCEEDED, + TASK_STATUS_FAILED, TASK_STATUS_RETRYING) + +import cherrypy + +# Suite status strings. +SUITE_STATUS_HELD = "held" +SUITE_STATUS_RUNNING = "running" +SUITE_STATUS_STOPPING = "stopping" +SUITE_STATUS_RUNNING_TO_STOP = "running to stop at %s" +SUITE_STATUS_RUNNING_TO_HOLD = "running to hold at %s" +# Regex to extract the stop or hold point. +SUITE_STATUS_SPLIT_REC = re.compile('^([a-z ]+ at )(.*)$') + +# Pseudo status strings for use by suite monitors. +# Use before attempting to determine status: +SUITE_STATUS_NOT_CONNECTED = "not connected" +# Use prior to first status update: +SUITE_STATUS_CONNECTED = "connected" +SUITE_STATUS_INITIALISING = "initialising" +# Use when the suite is not running: +SUITE_STATUS_STOPPED = "stopped" +SUITE_STATUS_STOPPED_WITH = "stopped with '%s'" + + +def get_suite_status_string(paused, stopping, will_pause_at, will_stop_at): + """Construct a suite status summary string for client programs. + + This is in a function for re-use in monitor and GUI back-compat code + (clients at cylc version <= 6.9.1 construct their own status string). + + """ + if paused: + return SUITE_STATUS_HELD + elif stopping: + return SUITE_STATUS_STOPPING + elif will_pause_at: + return SUITE_STATUS_RUNNING_TO_HOLD % will_pause_at + elif will_stop_at: + return SUITE_STATUS_RUNNING_TO_STOP % will_stop_at + else: + return SUITE_STATUS_RUNNING + + +class SuiteStillInitialisingError(Exception): + """Exception raised if a summary is requested before the first update. + + This can happen if client connects during start-up for large suites. + + """ + def __str__(self): + return "Suite initializing..." + + +class StateSummaryServer(BaseCommsServer): + """Server-side suite state summary interface.""" + + _INSTANCE = None + + @classmethod + def get_inst(cls, run_mode=None): + """Return a singleton instance.""" + if cls._INSTANCE is None: + cls._INSTANCE = cls(run_mode) + return cls._INSTANCE + + def __init__(self, run_mode): + super(StateSummaryServer, self).__init__() + self.task_summary = {} + self.global_summary = {} + self.family_summary = {} + self.run_mode = run_mode + self.first_update_completed = False + self._summary_update_time = None + + self.state_count_totals = {} + self.state_count_cycles = {} + + def update(self, tasks, tasks_rh, min_point, max_point, max_point_rh, + paused, will_pause_at, stopping, will_stop_at, ns_defn_order, + reloading): + global_summary = {} + family_summary = {} + + task_summary, task_states = self._get_tasks_info(tasks, tasks_rh) + + fam_states = {} + all_states = [] + config = SuiteConfig.get_inst() + ancestors_dict = config.get_first_parent_ancestors() + + # Compute state_counts (total, and per cycle). + state_count_totals = {} + state_count_cycles = {} + + for point_string, c_task_states in task_states: + # For each cycle point, construct a family state tree + # based on the first-parent single-inheritance tree + + c_fam_task_states = {} + + count = {} + + for key in c_task_states: + state = c_task_states[key] + if state is None: + continue + try: + count[state] += 1 + except KeyError: + count[state] = 1 + + all_states.append(state) + for parent in ancestors_dict[key]: + if parent == key: + continue + c_fam_task_states.setdefault(parent, set([])) + c_fam_task_states[parent].add(state) + + state_count_cycles[point_string] = count + + for fam, child_states in c_fam_task_states.items(): + f_id = TaskID.get(fam, point_string) + state = extract_group_state(child_states) + if state is None: + continue + try: + famcfg = config.cfg['runtime'][fam] + except KeyError: + famcfg = {} + description = famcfg.get('description') + title = famcfg.get('title') + family_summary[f_id] = {'name': fam, + 'description': description, + 'title': title, + 'label': point_string, + 'state': state} + + state_count_totals = {} + for point_string, count in state_count_cycles.items(): + for state, state_count in count.items(): + state_count_totals.setdefault(state, 0) + state_count_totals[state] += state_count + + all_states.sort() + + global_summary['oldest cycle point string'] = ( + self.str_or_None(min_point)) + global_summary['newest cycle point string'] = ( + self.str_or_None(max_point)) + global_summary['newest runahead cycle point string'] = ( + self.str_or_None(max_point_rh)) + if cylc.flags.utc: + global_summary['daemon time zone info'] = TIME_ZONE_UTC_INFO + else: + global_summary['daemon time zone info'] = TIME_ZONE_LOCAL_INFO + global_summary['last_updated'] = time.time() + global_summary['run_mode'] = self.run_mode + global_summary['states'] = all_states + global_summary['namespace definition order'] = ns_defn_order + global_summary['reloading'] = reloading + global_summary['state totals'] = state_count_totals + + # Construct a suite status string for use by monitoring clients. + global_summary['status_string'] = get_suite_status_string( + paused, stopping, will_pause_at, will_stop_at) + + self._summary_update_time = time.time() + + # Replace the originals (atomic update, for access from other threads). + self.task_summary = task_summary + self.global_summary = global_summary + self.family_summary = family_summary + self.first_update_completed = True + self.state_count_totals = state_count_totals + self.state_count_cycles = state_count_cycles + + def _get_tasks_info(self, tasks, tasks_rh): + """Retrieve task summary info and states.""" + + task_summary = {} + task_states = {} + + for task in tasks: + ts = task.get_state_summary() + task_summary[task.identity] = ts + name, point_string = TaskID.split(task.identity) + task_states.setdefault(point_string, {}) + task_states[point_string][name] = ts['state'] + + for task in tasks_rh: + ts = task.get_state_summary() + ts['state'] = TASK_STATUS_RUNAHEAD + task_summary[task.identity] = ts + name, point_string = TaskID.split(task.identity) + task_states.setdefault(point_string, {}) + task_states[point_string][name] = TASK_STATUS_RUNAHEAD + + return task_summary, task_states.items() + + def str_or_None(self, s): + if s: + return str(s) + else: + return None + + def get_state_totals(self): + # (Access to this is controlled via the suite_identity server.) + return (self.state_count_totals, self.state_count_cycles) + + @cherrypy.expose + @cherrypy.tools.json_out() + def get_state_summary(self): + """Return the global, task, and family summary data structures.""" + check_access_priv(self, 'full-read') + self.report('get_state_summary') + if not self.first_update_completed: + raise SuiteStillInitialisingError() + print + print repr(self.global_summary) + print + print repr(self.task_summary) + print + print repr(self.family_summary) + print + return (self.global_summary, self.task_summary, self.family_summary) + + @cherrypy.expose + @cherrypy.tools.json_out() + def get_summary_update_time(self): + """Return the last time the summaries were changed (Unix time).""" + check_access_priv(self, 'state-totals') + self.report('get_state_summary_update_time') + if not self.first_update_completed: + raise SuiteStillInitialisingError() + return self._summary_update_time + + +class StateSummaryClient(BaseCommsClient): + """Client-side suite state summary interface.""" + + def get_suite_state_summary(self): + return self.call_server_func("state", "get_state_summary") + + def get_suite_state_summary_update_time(self): + return self.call_server_func("state", "get_summary_update_time") + + +def extract_group_state(child_states, is_stopped=False): + """Summarise child states as a group.""" + + ordered_states = [TASK_STATUS_SUBMIT_FAILED, TASK_STATUS_FAILED, + TASK_STATUS_EXPIRED, TASK_STATUS_SUBMIT_RETRYING, + TASK_STATUS_RETRYING, TASK_STATUS_RUNNING, + TASK_STATUS_SUBMITTED, TASK_STATUS_READY, + TASK_STATUS_QUEUED, TASK_STATUS_WAITING, + TASK_STATUS_HELD, TASK_STATUS_SUCCEEDED, + TASK_STATUS_RUNAHEAD] + if is_stopped: + ordered_states = [TASK_STATUS_SUBMIT_FAILED, TASK_STATUS_FAILED, + TASK_STATUS_RUNNING, TASK_STATUS_SUBMITTED, + TASK_STATUS_EXPIRED, TASK_STATUS_READY, + TASK_STATUS_SUBMIT_RETRYING, TASK_STATUS_RETRYING, + TASK_STATUS_SUCCEEDED, TASK_STATUS_QUEUED, + TASK_STATUS_WAITING, TASK_STATUS_HELD, + TASK_STATUS_RUNAHEAD] + for state in ordered_states: + if state in child_states: + return state + return None + + +def get_id_summary(id_, task_state_summary, fam_state_summary, id_family_map): + """Return some state information about a task or family id.""" + prefix_text = "" + meta_text = "" + sub_text = "" + sub_states = {} + stack = [(id_, 0)] + done_ids = [] + for summary in [task_state_summary, fam_state_summary]: + if id_ in summary: + title = summary[id_].get('title') + if title: + meta_text += "\n" + title.strip() + description = summary[id_].get('description') + if description: + meta_text += "\n" + description.strip() + while stack: + this_id, depth = stack.pop(0) + if this_id in done_ids: # family dive down will give duplicates + continue + done_ids.append(this_id) + prefix = "\n" + " " * 4 * depth + this_id + if this_id in task_state_summary: + submit_num = task_state_summary[this_id].get('submit_num') + if submit_num: + prefix += "(%02d)" % submit_num + state = task_state_summary[this_id]['state'] + sub_text += prefix + " " + state + sub_states.setdefault(state, 0) + sub_states[state] += 1 + elif this_id in fam_state_summary: + name, point_string = TaskID.split(this_id) + sub_text += prefix + " " + fam_state_summary[this_id]['state'] + for child in reversed(sorted(id_family_map[name])): + child_id = TaskID.get(child, point_string) + stack.insert(0, (child_id, depth + 1)) + if not prefix_text: + prefix_text = sub_text.strip() + sub_text = "" + if len(sub_text.splitlines()) > 10: + state_items = sub_states.items() + state_items.sort() + state_items.sort(lambda x, y: cmp(y[1], x[1])) + sub_text = "" + for state, number in state_items: + sub_text += "\n {0} tasks {1}".format(number, state) + if sub_text and meta_text: + sub_text = "\n" + sub_text + text = prefix_text + meta_text + sub_text + if not text: + return id_ + return text diff --git a/lib/cylc/network/https/task_msgqueue.py b/lib/cylc/network/https/task_msgqueue.py new file mode 100644 index 00000000000..2251c34c1b2 --- /dev/null +++ b/lib/cylc/network/https/task_msgqueue.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2016 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from Queue import Queue +from cylc.owner import USER +from cylc.suite_host import get_hostname +from cylc.network import check_access_priv +from cylc.network.https.base import BaseCommsServer, BaseCommsClient +from cylc.registration import RegistrationDB + +import cherrypy +import os +import requests + + +class TaskMessageServer(BaseCommsServer): + """Server-side task messaging interface""" + + def __init__(self, suite): + self.queue = Queue() + super(TaskMessageServer, self).__init__() + + @cherrypy.expose + @cherrypy.tools.json_out() + def put(self, task_id, priority, message): + check_access_priv(self, 'full-control') + self.report('task_message') + self.queue.put((task_id, priority, str(message))) + return 'Message queued' + + def get_queue(self): + return self.queue + + +class TaskMessageClient(BaseCommsClient): + """Client-side task messaging interface""" + + def __init__(self, suite, task_id, owner=USER, + host=get_hostname(), timeout=None, port=None): + self.task_id = task_id + super(TaskMessageClient, self).__init__(suite, owner, host, + port=port, timeout=timeout) + + def put(self, priority, message): + return self.call_server_func("message", "put", + task_id=self.task_id, + priority=priority, + message=message) diff --git a/lib/cylc/network/https/util.py b/lib/cylc/network/https/util.py new file mode 100644 index 00000000000..6292952cb3e --- /dev/null +++ b/lib/cylc/network/https/util.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2016 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Utility classes for HTTPS servers and clients.""" + +import HTMLParser + + +class ExceptionPreReader(HTMLParser.HTMLParser): + + def __init__(self): + self.is_in_traceback_pre = False + self.exception_text = None + # Can't use super because this is an old-style class... :( + HTMLParser.HTMLParser.__init__(self) + + def handle_starttag(self, tag, attributes): + if tag != "pre": + return + for name, value in attributes: + if name == "id" and value == "traceback": + self.is_in_traceback_pre = True + + def handle_endtag(self, tag): + self.is_in_traceback_pre = False + + def handle_data(self, data): + if hasattr(self, "is_in_traceback_pre") and self.is_in_traceback_pre: + if self.exception_text is None: + self.exception_text = "" + self.exception_text += data + + +def get_exception_from_html(html_text): + """Return any content inside a 
 block with id 'traceback', or None.
+
+    Return e.g. 'abcdef' for text like '
+    abcdef
+    
'.
+
+    """
+    parser = ExceptionPreReader()
+    try:
+        parser.feed(parser.unescape(html_text))
+        parser.close()
+    except HTMLParser.HTMLParseError:
+        return None
+    return parser.exception_text
+
+
+def unicode_encode(data):
+    if isinstance(data, unicode):
+        return data.encode('utf-8')
+    if isinstance(data, dict):
+        new_dict = {}
+        for key, value in data.items():
+            new_dict.update(
+                {unicode_encode(key): unicode_encode(value)}
+            )
+        return new_dict
+    if isinstance(data, list):
+        return [unicode_encode(item) for item in data]
+    return data
diff --git a/lib/cylc/network/method.py b/lib/cylc/network/method.py
new file mode 100644
index 00000000000..7356bb73984
--- /dev/null
+++ b/lib/cylc/network/method.py
@@ -0,0 +1,18 @@
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+
+"""Package for network interfaces to cylc suite server objects."""
+
+from cylc.cfgspec.globalcfg import GLOBAL_CFG
+
+METHOD = GLOBAL_CFG.get(['communication', 'method'])
diff --git a/lib/cylc/network/port_scan.py b/lib/cylc/network/port_scan.py
index 100e9842caf..3538dc200e5 100644
--- a/lib/cylc/network/port_scan.py
+++ b/lib/cylc/network/port_scan.py
@@ -1,4 +1,4 @@
-#!/usr/bin/pyro
+#!/usr/bin/env python
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2016 NIWA
@@ -15,168 +15,10 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see .
-"""Pyro port scan utilities."""
+"""Wrap communications daemon for a suite."""
 
-from multiprocessing import cpu_count, Pool
-import sys
-from time import sleep
-import traceback
+from cylc.network.method import METHOD
 
-import Pyro.errors
-import Pyro.core
 
-from cylc.cfgspec.globalcfg import GLOBAL_CFG
-import cylc.flags
-from cylc.network import PYRO_SUITEID_OBJ_NAME, NO_PASSPHRASE
-from cylc.network.connection_validator import ConnValidator, SCAN_HASH
-from cylc.network.suite_state import SuiteStillInitialisingError
-from cylc.owner import USER
-from cylc.registration import RegistrationDB
-from cylc.suite_host import get_hostname, is_remote_host
-
-
-def get_proxy(host, port, pyro_timeout):
-    """Return Pyro URL of proxy."""
-    proxy = Pyro.core.getProxyForURI(
-        'PYROLOC://%s:%s/%s' % (host, port, PYRO_SUITEID_OBJ_NAME))
-    proxy._setTimeout(pyro_timeout)
-    return proxy
-
-
-def scan(host=None, db=None, pyro_timeout=None):
-    """Scan ports, return a list of suites found: [(port, suite.identify())].
-
-    Note that we could easily scan for a given suite+owner and return its
-    port instead of reading port files, but this may not always be fast enough.
-    """
-    if host is None:
-        host = get_hostname()
-    base_port = GLOBAL_CFG.get(['pyro', 'base port'])
-    last_port = base_port + GLOBAL_CFG.get(['pyro', 'maximum number of ports'])
-    if pyro_timeout:
-        pyro_timeout = float(pyro_timeout)
-    else:
-        pyro_timeout = None
-
-    reg_db = RegistrationDB(db)
-    results = []
-    for port in range(base_port, last_port):
-        try:
-            proxy = get_proxy(host, port, pyro_timeout)
-            conn_val = ConnValidator()
-            conn_val.set_default_hash(SCAN_HASH)
-            proxy._setNewConnectionValidator(conn_val)
-            proxy._setIdentification((USER, NO_PASSPHRASE))
-            result = (port, proxy.identify())
-        except Pyro.errors.ConnectionDeniedError as exc:
-            if cylc.flags.debug:
-                print '%s:%s (connection denied)' % (host, port)
-            # Back-compat <= 6.4.1
-            msg = '  Old daemon at %s:%s?' % (host, port)
-            for pphrase in reg_db.load_all_passphrases():
-                try:
-                    proxy = get_proxy(host, port, pyro_timeout)
-                    proxy._setIdentification(pphrase)
-                    info = proxy.id()
-                    result = (port, {'name': info[0], 'owner': info[1]})
-                except Pyro.errors.ConnectionDeniedError:
-                    connected = False
-                else:
-                    connected = True
-                    break
-            if not connected:
-                if cylc.flags.verbose:
-                    print >> sys.stderr, msg, "- connection denied (%s)" % exc
-                continue
-            else:
-                if cylc.flags.verbose:
-                    print >> sys.stderr, msg, "- connected with passphrase"
-        except Pyro.errors.TimeoutError as exc:
-            # E.g. Ctrl-Z suspended suite - holds up port scanning!
-            if cylc.flags.debug:
-                print '%s:%s (connection timed out)' % (host, port)
-            print >> sys.stderr, (
-                'suite? owner?@%s:%s - connection timed out (%s)' % (
-                    host, port, exc))
-            continue
-        except (Pyro.errors.ProtocolError, Pyro.errors.NamingError) as exc:
-            # No suite at this port.
-            if cylc.flags.debug:
-                print str(exc)
-                print '%s:%s (no suite)' % (host, port)
-            continue
-        except SuiteStillInitialisingError:
-            continue
-        except Exception as exc:
-            if cylc.flags.debug:
-                traceback.print_exc()
-            raise
-        else:
-            owner = result[1].get('owner')
-            name = result[1].get('name')
-            states = result[1].get('states', None)
-            if cylc.flags.debug:
-                print '   suite:', name, owner
-            if states is None:
-                # This suite keeps its state info private.
-                # Try again with the passphrase if I have it.
-                pphrase = reg_db.load_passphrase(name, owner, host)
-                if pphrase:
-                    try:
-                        proxy = get_proxy(host, port, pyro_timeout)
-                        conn_val = ConnValidator()
-                        conn_val.set_default_hash(SCAN_HASH)
-                        proxy._setNewConnectionValidator(conn_val)
-                        proxy._setIdentification((USER, pphrase))
-                        result = (port, proxy.identify())
-                    except Exception:
-                        # Nope (private suite, wrong passphrase).
-                        if cylc.flags.debug:
-                            print '    (wrong passphrase)'
-                    else:
-                        reg_db.cache_passphrase(
-                            name, owner, host, pphrase)
-                        if cylc.flags.debug:
-                            print '    (got states with passphrase)'
-        results.append(result)
-    return results
-
-
-def scan_all(hosts=None, reg_db_path=None, pyro_timeout=None):
-    """Scan all hosts."""
-    if not hosts:
-        hosts = GLOBAL_CFG.get(["suite host scanning", "hosts"])
-    # Ensure that it does "localhost" only once
-    hosts = set(hosts)
-    for host in list(hosts):
-        if not is_remote_host(host):
-            hosts.remove(host)
-            hosts.add("localhost")
-    proc_pool_size = GLOBAL_CFG.get(["process pool size"])
-    if proc_pool_size is None:
-        proc_pool_size = cpu_count()
-    if proc_pool_size > len(hosts):
-        proc_pool_size = len(hosts)
-    proc_pool = Pool(proc_pool_size)
-    async_results = {}
-    for host in hosts:
-        async_results[host] = proc_pool.apply_async(
-            scan, [host, reg_db_path, pyro_timeout])
-    proc_pool.close()
-    scan_results = []
-    scan_results_hosts = []
-    while async_results:
-        sleep(0.05)
-        for host, async_result in async_results.items():
-            if async_result.ready():
-                async_results.pop(host)
-                try:
-                    res = async_result.get()
-                except Exception:
-                    if cylc.flags.debug:
-                        traceback.print_exc()
-                else:
-                    scan_results.extend(res)
-                    scan_results_hosts.extend([host] * len(res))
-    proc_pool.join()
-    return zip(scan_results_hosts, scan_results)
+if METHOD == "https":
+    from cylc.network.https.port_scan import scan_all
diff --git a/lib/cylc/network/pyro_base.py b/lib/cylc/network/pyro_base.py
deleted file mode 100644
index a11fb816eea..00000000000
--- a/lib/cylc/network/pyro_base.py
+++ /dev/null
@@ -1,221 +0,0 @@
-#!/usr/bin/env python
-
-# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
-# Copyright (C) 2008-2016 NIWA
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
-"""Base classes for Pyro servers and clients."""
-
-import os
-import shlex
-from subprocess import Popen, PIPE
-import sys
-import traceback
-from uuid import uuid4
-
-import Pyro.core
-import Pyro.errors
-
-from cylc.cfgspec.globalcfg import GLOBAL_CFG
-from cylc.exceptions import PortFileError
-import cylc.flags
-from cylc.network.client_reporter import PyroClientReporter
-from cylc.network.connection_validator import ConnValidator, OK_HASHES
-from cylc.owner import is_remote_user, USER
-from cylc.registration import RegistrationDB
-from cylc.suite_host import get_hostname, is_remote_host
-from cylc.suite_env import CylcSuiteEnv, CylcSuiteEnvLoadError
-
-
-class PyroServer(Pyro.core.ObjBase):
-    """Base class for server-side suite object interfaces."""
-
-    def __init__(self):
-        Pyro.core.ObjBase.__init__(self)
-        self.client_reporter = PyroClientReporter.get_inst()
-
-    def signout(self):
-        """Wrap client_reporter.signout."""
-        self.client_reporter.signout(self)
-
-    def report(self, command):
-        """Wrap client_reporter.report."""
-        self.client_reporter.report(command, self)
-
-
-class PyroClient(object):
-    """Base class for client-side suite object interfaces."""
-
-    target_server_object = None
-
-    def __init__(self, suite, owner=USER, host=None, pyro_timeout=None,
-                 port=None, db=None, my_uuid=None, print_uuid=False):
-        self.suite = suite
-        self.host = host
-        self.owner = owner
-        if pyro_timeout is not None:
-            pyro_timeout = float(pyro_timeout)
-        self.pyro_timeout = pyro_timeout
-        self.port = port
-        self.pyro_proxy = None
-        self.my_uuid = my_uuid or uuid4()
-        self.uri = None
-        if print_uuid:
-            print >> sys.stderr, '%s' % self.my_uuid
-        self.reg_db = RegistrationDB(db)
-        self.pphrase = None
-
-    def call_server_func(self, fname, *fargs):
-        """Call server_object.fname(*fargs)
-
-        Get a Pyro proxy for the server object if we don't already have it,
-        and handle back compat retry for older daemons.
-
-        """
-        items = [
-            {},
-            {"reset": True, "cache_ok": False},
-            {"reset": True, "cache_ok": False, "old": True},
-        ]
-        for hash_name in OK_HASHES[1:]:
-            items.append(
-                {"reset": True, "cache_ok": False, "hash_name": hash_name})
-        for i, proxy_kwargs in enumerate(items):
-            func = getattr(self._get_proxy(**proxy_kwargs), fname)
-            try:
-                ret = func(*fargs)
-                break
-            except Pyro.errors.ProtocolError:
-                if i + 1 == len(items):  # final attempt
-                    raise
-        self.reg_db.cache_passphrase(
-            self.suite, self.owner, self.host, self.pphrase)
-        return ret
-
-    def _set_uri(self):
-        """Set Pyro URI.
-
-        Determine host and port using content in port file, unless already
-        specified.
-
-        """
-        if ((self.host is None or self.port is None) and
-                'CYLC_SUITE_RUN_DIR' in os.environ):
-            # Looks like we are in a running task job, so we should be able to
-            # use "cylc-suite-env" file under the suite running directory
-            try:
-                suite_env = CylcSuiteEnv.load(
-                    self.suite, os.environ['CYLC_SUITE_RUN_DIR'])
-            except CylcSuiteEnvLoadError:
-                if cylc.flags.debug:
-                    traceback.print_exc()
-            else:
-                self.host = suite_env.suite_host
-                self.port = suite_env.suite_port
-                self.owner = suite_env.suite_owner
-
-        if self.host is None or self.port is None:
-            port_file_path = os.path.join(
-                GLOBAL_CFG.get(['pyro', 'ports directory']), self.suite)
-            if is_remote_host(self.host) or is_remote_user(self.owner):
-                ssh_tmpl = str(GLOBAL_CFG.get_host_item(
-                    'remote shell template', self.host, self.owner))
-                ssh_tmpl = ssh_tmpl.replace(' %s', '')
-                user_at_host = ''
-                if self.owner:
-                    user_at_host = self.owner + '@'
-                if self.host:
-                    user_at_host += self.host
-                else:
-                    user_at_host += 'localhost'
-                r_port_file_path = port_file_path.replace(
-                    os.environ['HOME'], '$HOME')
-                command = shlex.split(ssh_tmpl) + [
-                    user_at_host, 'cat', r_port_file_path]
-                proc = Popen(command, stdout=PIPE, stderr=PIPE)
-                out, err = proc.communicate()
-                ret_code = proc.wait()
-                if ret_code:
-                    if cylc.flags.debug:
-                        print >> sys.stderr, {
-                            "code": ret_code,
-                            "command": command,
-                            "stdout": out,
-                            "stderr": err}
-                    raise PortFileError(
-                        "Port file '%s:%s' not found - suite not running?." %
-                        (user_at_host, r_port_file_path))
-            else:
-                try:
-                    out = open(port_file_path).read()
-                except IOError:
-                    raise PortFileError(
-                        "Port file '%s' not found - suite not running?." %
-                        (port_file_path))
-            lines = out.splitlines()
-            try:
-                if self.port is None:
-                    self.port = int(lines[0])
-            except (IndexError, ValueError):
-                raise PortFileError(
-                    "ERROR, bad content in port file: %s" % port_file_path)
-            if self.host is None:
-                if len(lines) >= 2:
-                    self.host = lines[1].strip()
-                else:
-                    self.host = get_hostname()
-
-        # Qualify the obj name with user and suite name (unnecessary but
-        # can't change it until we break back-compat with older daemons).
-        self.uri = (
-            'PYROLOC://%(host)s:%(port)s/%(owner)s.%(suite)s.%(target)s' % {
-                "host": self.host,
-                "port": self.port,
-                "suite": self.suite,
-                "owner": self.owner,
-                "target": self.target_server_object})
-
-    def _get_proxy(self, reset=True, hash_name=None, cache_ok=True, old=False):
-        """Get a Pyro proxy."""
-        if reset or self.pyro_proxy is None:
-            self._set_uri()
-            self.pphrase = self.reg_db.load_passphrase(
-                self.suite, self.owner, self.host, cache_ok)
-            # Fails only for unknown hosts (no connection till RPC call).
-            self.pyro_proxy = Pyro.core.getProxyForURI(self.uri)
-            self.pyro_proxy._setTimeout(self.pyro_timeout)
-            if old:
-                self.pyro_proxy._setIdentification(self.pphrase)
-            else:
-                conn_val = ConnValidator()
-                if hash_name is None:
-                    hash_name = getattr(self, "_hash_name", None)
-                if hash_name is not None and hash_name in OK_HASHES:
-                    conn_val.set_default_hash(hash_name)
-                self.pyro_proxy._setNewConnectionValidator(conn_val)
-                self.pyro_proxy._setIdentification(
-                    (self.my_uuid, self.pphrase))
-        return self.pyro_proxy
-
-    def reset(self):
-        """Reset pyro_proxy."""
-        self.pyro_proxy = None
-
-    def signout(self):
-        """Multi-connect clients should call this on exit."""
-        try:
-            self._get_proxy().signout()
-        except Exception:
-            # Suite may have stopped before the client exits.
-            pass
diff --git a/lib/cylc/network/pyro_daemon.py b/lib/cylc/network/pyro_daemon.py
deleted file mode 100644
index 1f3bef44457..00000000000
--- a/lib/cylc/network/pyro_daemon.py
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/usr/bin/env python
-
-# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
-# Copyright (C) 2008-2016 NIWA
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
-"""Wrap Pyro daemon for a suite."""
-
-import socket
-import traceback
-
-import Pyro
-from cylc.cfgspec.globalcfg import GLOBAL_CFG
-from cylc.network.connection_validator import ConnValidator
-from cylc.owner import USER
-from cylc.registration import RegistrationDB
-
-
-class PyroDaemon(object):
-    """Wrap Pyro daemon for a suite."""
-
-    def __init__(self, suite, suite_dir):
-        # Suite only needed for back-compat with old clients (see below):
-        self.suite = suite
-
-        Pyro.config.PYRO_MULTITHREADED = 1
-        # Use dns names instead of fixed ip addresses from /etc/hosts
-        # (see the Userguide "Networking Issues" section).
-        Pyro.config.PYRO_DNS_URI = True
-
-        # Base Pyro socket number.
-        Pyro.config.PYRO_PORT = GLOBAL_CFG.get(['pyro', 'base port'])
-        # Max number of sockets starting at base.
-        Pyro.config.PYRO_PORT_RANGE = GLOBAL_CFG.get(
-            ['pyro', 'maximum number of ports'])
-
-        Pyro.core.initServer()
-        self.daemon = Pyro.core.Daemon()
-        cval = ConnValidator()
-        self.daemon.setNewConnectionValidator(cval)
-        cval.set_pphrase(RegistrationDB.load_passphrase_from_dir(suite_dir))
-
-    def shutdown(self):
-        """Shutdown the daemon."""
-        self.daemon.shutdown(True)
-        # If a suite shuts down via 'stop --now' or # Ctrl-C, etc.,
-        # any existing client end connections will hang for a long time
-        # unless we do the following (or cylc clients set a timeout,
-        # presumably) which daemon.shutdown() does not do (why not?):
-
-        try:
-            self.daemon.sock.shutdown(socket.SHUT_RDWR)
-        except socket.error:
-            traceback.print_exc()
-
-        # Force all Pyro threads to stop now, to prevent them from raising any
-        # exceptions during Python interpreter shutdown - see GitHub #1890.
-        self.daemon.closedown()
-
-    def connect(self, obj, name):
-        """Connect obj and name to the daemon."""
-        if not obj.__class__.__name__ == 'SuiteIdServer':
-            # Qualify the obj name with user and suite name (unnecessary but
-            # can't change it until we break back-compat with older daemons).
-            name = "%s.%s.%s" % (USER, self.suite, name)
-        self.daemon.connect(obj, name)
-
-    def disconnect(self, obj):
-        """Disconnect obj from the daemon."""
-        self.daemon.disconnect(obj)
-
-    def handle_requests(self, timeout=None):
-        """Handle Pyro requests."""
-        self.daemon.handleRequests(timeout)
-
-    def get_port(self):
-        """Return the daemon port."""
-        return self.daemon.port
diff --git a/lib/cylc/network/suite_broadcast.py b/lib/cylc/network/suite_broadcast.py
index 164d07070a9..78132c4aaf3 100644
--- a/lib/cylc/network/suite_broadcast.py
+++ b/lib/cylc/network/suite_broadcast.py
@@ -15,388 +15,11 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see .
+"""Wrap communications daemon for a suite."""
 
-import sys
-import logging
-import cPickle as pickle
-import threading
+from cylc.network.method import METHOD
 
-from cylc.broadcast_report import (
-    get_broadcast_change_iter,
-    get_broadcast_change_report,
-    get_broadcast_bad_options_report)
-from cylc.cycling.loader import get_point, standardise_point_string
-from cylc.wallclock import get_current_time_string
-from cylc.network import PYRO_BCAST_OBJ_NAME
-from cylc.network.pyro_base import PyroClient, PyroServer
-from cylc.network import check_access_priv
-from cylc.task_id import TaskID
-from cylc.rundb import CylcSuiteDAO
 
-
-class BroadcastServer(PyroServer):
-    """Server-side suite broadcast interface.
-
-    Examples:
-    self.settings['*']['root'] = {'environment': {'FOO': 'bar'}}
-    self.settings['20100808T06Z']['root'] = {'command scripting': 'stuff'}
-    """
-
-    _INSTANCE = None
-    ALL_CYCLE_POINTS_STRS = ["*", "all-cycle-points", "all-cycles"]
-    TABLE_BROADCAST_EVENTS = CylcSuiteDAO.TABLE_BROADCAST_EVENTS
-    TABLE_BROADCAST_STATES = CylcSuiteDAO.TABLE_BROADCAST_STATES
-
-    @classmethod
-    def get_inst(cls, linearized_ancestors=None):
-        """Return a singleton instance.
-
-        On 1st call, instantiate the singleton.
-        Argument linearized_ancestors is only relevant on 1st call.
-
-        """
-        if cls._INSTANCE is None:
-            cls._INSTANCE = cls(linearized_ancestors)
-        return cls._INSTANCE
-
-    def __init__(self, linearized_ancestors):
-        super(BroadcastServer, self).__init__()
-        self.log = logging.getLogger('main')
-        self.settings = {}
-        self.db_inserts_map = {
-            self.TABLE_BROADCAST_EVENTS: [],
-            self.TABLE_BROADCAST_STATES: []}
-        self.db_deletes_map = {
-            self.TABLE_BROADCAST_STATES: []}
-        self.linearized_ancestors = linearized_ancestors
-        self.lock = threading.RLock()
-
-    def _prune(self):
-        """Remove empty leaves left by unsetting broadcast values.
-
-        Return a list of pruned settings in the form:
-
-        [
-            ["20200202", "foo", "command scripting"],
-            ["20020202", "bar", "environment", "BAR"],
-        ]
-        """
-        with self.lock:
-            prunes = []
-            stuff_stack = [([], self.settings, True)]
-            while stuff_stack:
-                keys, stuff, is_new = stuff_stack.pop()
-                if is_new:
-                    stuff_stack.append((keys, stuff, False))
-                    for key, value in stuff.items():
-                        if isinstance(value, dict):
-                            stuff_stack.append((keys + [key], value, True))
-                else:
-                    for key, value in stuff.items():
-                        if value in [None, {}]:
-                            del stuff[key]
-                            prunes.append(keys + [key])
-            return prunes
-
-    def _addict(self, target, source):
-        """Recursively add source dict to target dict."""
-        for key, val in source.items():
-            if isinstance(val, dict):
-                if key not in target:
-                    target[key] = {}
-                self._addict(target[key], val)
-            else:
-                target[key] = source[key]
-
-    def put(self, point_strings, namespaces, settings):
-        """Add new broadcast settings (server side interface).
-
-        Return a tuple (modified_settings, bad_options) where:
-          modified_settings is list of modified settings in the form:
-            [("20200202", "foo", {"command scripting": "true"}, ...]
-          bad_options is as described in the docstring for self.clear().
-        """
-        check_access_priv(self, 'full-control')
-        self.report('broadcast_put')
-        modified_settings = []
-        bad_point_strings = []
-        bad_namespaces = []
-
-        with self.lock:
-            for setting in settings:
-                for point_string in point_strings:
-                    # Standardise the point and check its validity.
-                    bad_point = False
-                    try:
-                        point_string = standardise_point_string(point_string)
-                    except Exception as exc:
-                        if point_string != '*':
-                            bad_point_strings.append(point_string)
-                            bad_point = True
-                    if not bad_point and point_string not in self.settings:
-                        self.settings[point_string] = {}
-                    for namespace in namespaces:
-                        if namespace not in self.linearized_ancestors:
-                            bad_namespaces.append(namespace)
-                        elif not bad_point:
-                            if namespace not in self.settings[point_string]:
-                                self.settings[point_string][namespace] = {}
-                            self._addict(
-                                self.settings[point_string][namespace],
-                                setting)
-                            modified_settings.append(
-                                (point_string, namespace, setting))
-
-        # Log the broadcast
-        self._append_db_queue(modified_settings)
-        self.log.info(get_broadcast_change_report(modified_settings))
-
-        bad_options = {}
-        if bad_point_strings:
-            bad_options["point_strings"] = bad_point_strings
-        if bad_namespaces:
-            bad_options["namespaces"] = bad_namespaces
-        return modified_settings, bad_options
-
-    def get(self, task_id=None):
-        """Retrieve all broadcast variables that target a given task ID."""
-        check_access_priv(self, 'full-read')
-        self.report('broadcast_get')
-        if not task_id:
-            # all broadcast settings requested
-            return self.settings
-        name, point_string = TaskID.split(task_id)
-
-        ret = {}
-        # The order is:
-        #    all:root -> all:FAM -> ... -> all:task
-        # -> tag:root -> tag:FAM -> ... -> tag:task
-        for cycle in self.ALL_CYCLE_POINTS_STRS + [point_string]:
-            if cycle not in self.settings:
-                continue
-            for namespace in reversed(self.linearized_ancestors[name]):
-                if namespace in self.settings[cycle]:
-                    self._addict(ret, self.settings[cycle][namespace])
-        return ret
-
-    def expire(self, cutoff):
-        """Clear all settings targeting cycle points earlier than cutoff."""
-        point_strings = []
-        cutoff_point = None
-        if cutoff is not None:
-            cutoff_point = get_point(str(cutoff))
-        with self.lock:
-            for point_string in self.settings:
-                if cutoff_point is None or (
-                        point_string not in self.ALL_CYCLE_POINTS_STRS and
-                        get_point(point_string) < cutoff_point):
-                    point_strings.append(point_string)
-        if not point_strings:
-            return (None, {"expire": [cutoff]})
-        return self.clear(point_strings=point_strings)
-
-    def clear(self, point_strings=None, namespaces=None, cancel_settings=None):
-        """Clear settings globally, or for listed namespaces and/or points.
-
-        Return a tuple (modified_settings, bad_options), where:
-        * modified_settings is similar to the return value of the "put" method,
-          but for removed settings.
-        * bad_options is a dict in the form:
-              {"point_strings": ["20020202", ..."], ...}
-          The dict is only populated if there are options not associated with
-          previous broadcasts. The keys can be:
-          * point_strings: a list of bad point strings.
-          * namespaces: a list of bad namespaces.
-          * cancel: a list of tuples. Each tuple contains the keys of a bad
-            setting.
-        """
-        # If cancel_settings defined, only clear specific settings
-        cancel_keys_list = self._settings_to_keys_list(cancel_settings)
-
-        # Clear settings
-        modified_settings = []
-        with self.lock:
-            for point_string, point_string_settings in self.settings.items():
-                if point_strings and point_string not in point_strings:
-                    continue
-                for namespace, namespace_settings in (
-                        point_string_settings.items()):
-                    if namespaces and namespace not in namespaces:
-                        continue
-                    stuff_stack = [([], namespace_settings)]
-                    while stuff_stack:
-                        keys, stuff = stuff_stack.pop()
-                        for key, value in stuff.items():
-                            if isinstance(value, dict):
-                                stuff_stack.append((keys + [key], value))
-                            elif (not cancel_keys_list or
-                                    keys + [key] in cancel_keys_list):
-                                stuff[key] = None
-                                setting = {key: value}
-                                for rkey in reversed(keys):
-                                    setting = {rkey: setting}
-                                modified_settings.append(
-                                    (point_string, namespace, setting))
-
-        # Prune any empty branches
-        bad_options = self._get_bad_options(
-            self._prune(), point_strings, namespaces, cancel_keys_list)
-
-        # Log the broadcast
-        self._append_db_queue(modified_settings, is_cancel=True)
-        self.log.info(
-            get_broadcast_change_report(modified_settings, is_cancel=True))
-        if bad_options:
-            self.log.error(get_broadcast_bad_options_report(bad_options))
-
-        return (modified_settings, bad_options)
-
-    @staticmethod
-    def _settings_to_keys_list(settings):
-        """Return a list containing each setting dict keys as a list.
-
-        E.g. Each setting in settings may look like:
-        {"foo": {"bar": {"baz": 1}}}
-
-        An element of the returned list will look like:
-        ["foo", "bar", "baz"]
-
-        """
-        keys_list = []
-        if settings:
-            for setting in settings:
-                stuff_stack = [([], setting)]
-                while stuff_stack:
-                    keys, stuff = stuff_stack.pop()
-                    for key, value in stuff.items():
-                        if isinstance(value, dict):
-                            stuff_stack.append((keys + [key], value))
-                        else:
-                            keys_list.append(keys + [key])
-        return keys_list
-
-    def dump(self, file_):
-        """Write broadcast variables to the state dump file."""
-        with self.lock:
-            pickle.dump(self.settings, file_)
-            file_.write("\n")
-
-    def load(self, pickled_settings):
-        """Load broadcast variables from the state dump file."""
-        with self.lock:
-            self.settings = pickle.loads(pickled_settings)
-
-            # Ensure database table is in sync
-            modified_settings = []
-            for point_string, point_string_settings in self.settings.items():
-                for namespace, namespace_settings in (
-                        point_string_settings.items()):
-                    stuff_stack = [([], namespace_settings)]
-                    while stuff_stack:
-                        keys, stuff = stuff_stack.pop()
-                        for key, value in stuff.items():
-                            if isinstance(value, dict):
-                                stuff_stack.append((keys + [key], value))
-                            else:
-                                setting = {key: value}
-                                for rkey in reversed(keys):
-                                    setting = {rkey: setting}
-                                modified_settings.append(
-                                    (point_string, namespace, setting))
-        for broadcast_change in get_broadcast_change_iter(modified_settings):
-            self.db_inserts_map[self.TABLE_BROADCAST_STATES].append({
-                "point": broadcast_change["point"],
-                "namespace": broadcast_change["namespace"],
-                "key": broadcast_change["key"],
-                "value": broadcast_change["value"]})
-
-    def _get_dump(self):
-        """Return broadcast variables as written to the state dump file."""
-        with self.lock:
-            return pickle.dumps(self.settings) + "\n"
-
-    @classmethod
-    def _get_bad_options(
-            cls, prunes, point_strings, namespaces, cancel_keys_list):
-        """Return unpruned namespaces and/or point_strings options."""
-        cancel_keys_list = [
-            tuple(cancel_keys) for cancel_keys in cancel_keys_list]
-        bad_options = {
-            "point_strings": None, "namespaces": None, "cancel": None}
-        # 1. Populate the bad_options dict where applicable.
-        # 2. Remove keys if they are found in "prunes".
-        # 3. Remove key in bad_options if it becomes empty.
-        for opt_name, opt_list, opt_test in [
-                ("point_strings", point_strings, cls._point_string_in_prunes),
-                ("namespaces", namespaces, cls._namespace_in_prunes),
-                ("cancel", cancel_keys_list, cls._cancel_keys_in_prunes)]:
-            if opt_list:
-                bad_options[opt_name] = set(opt_list)
-                for opt in opt_list:
-                    if opt_test(prunes, opt):
-                        bad_options[opt_name].discard(opt)
-                        if not bad_options[opt_name]:
-                            break
-        for key, value in bad_options.items():
-            if value:
-                bad_options[key] = list(value)
-            else:
-                del bad_options[key]
-        return bad_options
-
-    @staticmethod
-    def _point_string_in_prunes(prunes, point_string):
-        """Is point_string pruned?"""
-        return point_string in [prune[0] for prune in prunes]
-
-    @staticmethod
-    def _namespace_in_prunes(prunes, namespace):
-        """Is namespace pruned?"""
-        return namespace in [prune[1] for prune in prunes if prune[1:]]
-
-    @staticmethod
-    def _cancel_keys_in_prunes(prunes, cancel_keys):
-        """Is cancel_keys pruned?"""
-        return (list(cancel_keys) in
-                [prune[2:] for prune in prunes if prune[2:]])
-
-    def _append_db_queue(self, modified_settings, is_cancel=False):
-        """Update the queue to the runtime DB."""
-        now = get_current_time_string(display_sub_seconds=True)
-        for broadcast_change in (
-                get_broadcast_change_iter(modified_settings, is_cancel)):
-            broadcast_change["time"] = now
-            self.db_inserts_map[self.TABLE_BROADCAST_EVENTS].append(
-                broadcast_change)
-            if is_cancel:
-                self.db_deletes_map[self.TABLE_BROADCAST_STATES].append({
-                    "point": broadcast_change["point"],
-                    "namespace": broadcast_change["namespace"],
-                    "key": broadcast_change["key"]})
-                # Delete statements are currently executed before insert
-                # statements, so we should clear out any insert statements that
-                # are deleted here.
-                # (Not the most efficient logic here, but unless we have a
-                # large number of inserts, then this should not be a big
-                # concern.)
-                inserts = []
-                for insert in self.db_inserts_map[self.TABLE_BROADCAST_STATES]:
-                    if any([insert[key] != broadcast_change[key] for key in
-                            ["point", "namespace", "key"]]):
-                        inserts.append(insert)
-                self.db_inserts_map[self.TABLE_BROADCAST_STATES] = inserts
-            else:
-                self.db_inserts_map[self.TABLE_BROADCAST_STATES].append({
-                    "point": broadcast_change["point"],
-                    "namespace": broadcast_change["namespace"],
-                    "key": broadcast_change["key"],
-                    "value": broadcast_change["value"]})
-
-
-class BroadcastClient(PyroClient):
-    """Client-side suite broadcast interface."""
-
-    target_server_object = PYRO_BCAST_OBJ_NAME
-
-    def broadcast(self, cmd, *args):
-        return self.call_server_func(cmd, *args)
+if METHOD == "https":
+    from cylc.network.https.suite_broadcast import (
+        BroadcastServer, BroadcastClient)
diff --git a/lib/cylc/network/suite_command.py b/lib/cylc/network/suite_command.py
index 8f521472f39..556baac6501 100644
--- a/lib/cylc/network/suite_command.py
+++ b/lib/cylc/network/suite_command.py
@@ -15,76 +15,11 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see .
+"""Wrap communications daemon for a suite."""
 
-import sys
-import os
-from Queue import Queue
+from cylc.network.method import METHOD
 
-import cylc.flags
-from cylc.network import PYRO_CMD_OBJ_NAME
-from cylc.network.pyro_base import PyroClient, PyroServer
-from cylc.network import check_access_priv
 
-# Back-compat for older suite daemons <= 6.4.1.
-back_compat = {
-    'set_stop_cleanly': 'stop cleanly',
-    'stop_now': 'stop now',
-    'set_stop_after_point': 'stop after point',
-    'set_stop_after_clock_time': 'stop after clock time',
-    'set_stop_after_task': 'stop after task',
-    'release_suite': 'release suite',
-    'release_task': 'release task',
-    'remove_cycle': 'remove cycle',
-    'remove_task': 'remove task',
-    'hold_suite': 'hold suite now',
-    'hold_after_point_string': 'hold suite after',
-    'hold_task': 'hold task now',
-    'set_runahead': 'set runahead',
-    'set_verbosity': 'set verbosity',
-    'purge_tree': 'purge tree',
-    'reset_task_state': 'reset task state',
-    'trigger_task': 'trigger task',
-    'dry_run_task': 'dry run task',
-    'nudge': 'nudge suite',
-    'insert_task': 'insert task',
-    'reload_suite': 'reload suite',
-    'add_prerequisite': 'add prerequisite',
-    'poll_tasks': 'poll tasks',
-    'kill_tasks': 'kill tasks',
-    'spawn_tasks': 'reset_task_state -s spawn'
-}
-
-
-class SuiteCommandServer(PyroServer):
-    """Server-side suite command interface."""
-
-    def __init__(self):
-        super(SuiteCommandServer, self).__init__()
-        self.queue = Queue()
-
-    def put(self, command, *command_args):
-        if 'stop' in command:
-            check_access_priv(self, 'shutdown')
-        else:
-            check_access_priv(self, 'full-control')
-        self.report(command)
-        self.queue.put((command, command_args))
-        return (True, 'Command queued')
-
-    def get_queue(self):
-        return self.queue
-
-
-class SuiteCommandClient(PyroClient):
-    """Client-side suite command interface."""
-
-    target_server_object = PYRO_CMD_OBJ_NAME
-
-    def put_command(self, *args):
-        success, msg = self.call_server_func("put", *args)
-        if msg.startswith('ERROR: Illegal command:'):
-            # Back-compat for older suite daemons <= 6.4.1.
-            command = back_compat[args[0]]
-            args = tuple([command]) + args[1:]
-            success, msg = self.call_server_func("put", *args)
-        return (success, msg)
+if METHOD == "https":
+    from cylc.network.https.suite_command import (
+        SuiteCommandServer, SuiteCommandClient)
diff --git a/lib/cylc/network/suite_identifier.py b/lib/cylc/network/suite_identifier.py
index 4e33e141f1a..79df2dbaf1a 100644
--- a/lib/cylc/network/suite_identifier.py
+++ b/lib/cylc/network/suite_identifier.py
@@ -15,56 +15,11 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see .
+"""Wrap communications daemon for a suite."""
 
-# A minimal Pyro-connected object to allow client programs to identify
-# what suite is running at a given cylc port - by suite name and owner.
+from cylc.network.method import METHOD
 
-# All *other* suite objects should be connected to Pyro via qualified
-# names: owner.suite.object, to prevent accidental access to the wrong
-# suite. This object, however, should be connected unqualified so that
-# that same ID method can be called on any active cylc port.
 
-import cylc.flags
-from cylc.network.pyro_base import PyroServer
-from cylc.network.suite_state import StateSummaryServer
-from cylc.network import access_priv_ok
-from cylc.config import SuiteConfig
-
-
-class SuiteIdServer(PyroServer):
-    """Server-side external trigger interface."""
-
-    _INSTANCE = None
-
-    @classmethod
-    def get_inst(cls, name=None, owner=None):
-        """Return a singleton instance."""
-        if cls._INSTANCE is None:
-            cls._INSTANCE = cls(name, owner)
-        return cls._INSTANCE
-
-    def __init__(self, name, owner):
-        self.owner = owner
-        self.name = name
-        super(SuiteIdServer, self).__init__()
-
-    def identify(self):
-        self.report("identify")
-        result = {}
-        if access_priv_ok(self, "identity"):
-            result['name'] = self.name
-            result['owner'] = self.owner
-        if access_priv_ok(self, "description"):
-            config = SuiteConfig.get_inst()
-            result['title'] = config.cfg['title']
-            result['description'] = config.cfg['description']
-        if access_priv_ok(self, "state-totals"):
-            result['states'] = StateSummaryServer.get_inst().get_state_totals()
-            result['update-time'] = (
-                StateSummaryServer.get_inst().get_summary_update_time())
-        return result
-
-    def id(self):
-        # Back-compat for older clients <=6.4.1.
-        # (Allows old scan to see new suites.)
-        return (self.name, self.owner)
+if METHOD == "https":
+    from cylc.network.https.suite_identifier import (
+        SuiteIdServer, SuiteIdClientAnon)
diff --git a/lib/cylc/network/suite_info.py b/lib/cylc/network/suite_info.py
index 757cd22adea..9e971b9bcea 100644
--- a/lib/cylc/network/suite_info.py
+++ b/lib/cylc/network/suite_info.py
@@ -15,67 +15,11 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see .
+"""Wrap communications daemon for a suite."""
 
-import sys
+from cylc.network.method import METHOD
 
-import cylc.flags
-from cylc.network import PYRO_INFO_OBJ_NAME
-from cylc.network.pyro_base import PyroClient, PyroServer
-from cylc.network import check_access_priv
-from cylc.network.connection_validator import SCAN_HASH
 
-
-# Back-compat for older suite daemons <= 6.4.1.
-back_compat = {
-    'ping_suite': 'ping suite',
-    'ping_task': 'ping task',
-    'get_suite_info': 'suite info',
-    'get_task_info': 'task info',
-    'get_all_families': 'all families',
-    'get_triggering_families': 'triggering families',
-    'get_first_parent_ancestors': 'first-parent ancestors',
-    'get_first_parent_descendants': 'first-parent descendants',
-    'get_graph_raw': 'graph raw',
-    'get_task_requisites': 'task requisites',
-    'get_cylc_version': 'get cylc version',
-    'get_task_jobfile_path': 'task job file path'
-}
-
-
-class SuiteInfoServer(PyroServer):
-    """Server-side suite information interface."""
-
-    def __init__(self, info_commands):
-        super(SuiteInfoServer, self).__init__()
-        self.commands = info_commands
-
-    def get(self, command, *command_args):
-        if ('ping' in command or 'version' in command):
-            # Free info.
-            pass
-        elif 'suite' in command and 'info' in command:
-            # Suite title and description only.
-            check_access_priv(self, 'description')
-        else:
-            check_access_priv(self, 'full-read')
-        self.report(command)
-        return self.commands[command](*command_args)
-
-
-class SuiteInfoClient(PyroClient):
-    """Client-side suite information interface."""
-
-    target_server_object = PYRO_INFO_OBJ_NAME
-
-    def get_info(self, *args):
-        try:
-            return self.call_server_func("get", *args)
-        except KeyError:
-            # Back-compat for older suite daemons <= 6.4.1.
-            command = back_compat[args[0]]
-            args = tuple([command]) + args[1:]
-            return self.call_server_func("get", *args)
-
-    def set_use_scan_hash(self):
-        """Use the configured scan hash for backwards compatibility."""
-        self._hash_name = SCAN_HASH
+if METHOD == "https":
+    from cylc.network.https.suite_info import (
+        SuiteInfoServer, SuiteInfoClient, SuiteInfoClientAnon)
diff --git a/lib/cylc/network/suite_log.py b/lib/cylc/network/suite_log.py
index 18174707c36..d09335a22f9 100644
--- a/lib/cylc/network/suite_log.py
+++ b/lib/cylc/network/suite_log.py
@@ -15,59 +15,11 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see .
+"""Wrap communications daemon for a suite."""
 
-import os
-from cylc.network import PYRO_LOG_OBJ_NAME
-from cylc.network.pyro_base import PyroClient, PyroServer
-from cylc.network import check_access_priv
+from cylc.network.method import METHOD
 
 
-class SuiteLogServer(PyroServer):
-    """Server-side suite log interface."""
-
-    def __init__(self, log):
-        super(SuiteLogServer, self).__init__()
-        self.log = log
-        self.err_file = log.get_err_path()
-
-    def _get_err_has_changed(self, prev_err_size):
-        """Return True if the file has changed size compared to prev_size."""
-        return self._get_err_size() != prev_err_size
-
-    def _get_err_size(self):
-        """Return the os.path.getsize result for the error file."""
-
-        try:
-            size = os.path.getsize(self.err_file)
-        except (IOError, OSError) as e:
-            self.log.warning("Could not read suite err log file: %s" % e)
-            return 0
-        return size
-
-    def get_err_content(self, prev_size=0, max_lines=100):
-        """Return the content and new size of the error file."""
-
-        check_access_priv(self, 'full-read')
-        self.report("get_err_content")
-        if not self._get_err_has_changed(prev_size):
-            return [], prev_size
-        try:
-            f = open(self.err_file, "r")
-            f.seek(prev_size)
-            new_content = f.read()
-            f.close()
-            size = self._get_err_size()
-        except (IOError, OSError) as e:
-            self.log.warning("Could not read suite err log file: %s" % e)
-            return "", prev_size
-        new_content_lines = new_content.splitlines()[-max_lines:]
-        return "\n".join(new_content_lines), size
-
-
-class SuiteLogClient(PyroClient):
-    """Client-side suite log interface."""
-
-    target_server_object = PYRO_LOG_OBJ_NAME
-
-    def get_err_content(self, *args):
-        return self.call_server_func("get_err_content", *args)
+if METHOD == "https":
+    from cylc.network.https.suite_log import (
+        SuiteLogServer, SuiteLogClient)
diff --git a/lib/cylc/network/suite_state.py b/lib/cylc/network/suite_state.py
index 26e36d4779e..3085f5bb97e 100644
--- a/lib/cylc/network/suite_state.py
+++ b/lib/cylc/network/suite_state.py
@@ -15,341 +15,16 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see .
-
-import re
-import time
-import datetime
-
-import cylc.flags
-from cylc.task_id import TaskID
-from cylc.wallclock import TIME_ZONE_LOCAL_INFO, TIME_ZONE_UTC_INFO
-from cylc.config import SuiteConfig
-from cylc.network import PYRO_STATE_OBJ_NAME
-from cylc.network.pyro_base import PyroClient, PyroServer
-from cylc.network import check_access_priv
-from cylc.task_state import (
-    TASK_STATUS_RUNAHEAD, TASK_STATUS_HELD, TASK_STATUS_WAITING,
-    TASK_STATUS_EXPIRED, TASK_STATUS_QUEUED, TASK_STATUS_READY,
-    TASK_STATUS_SUBMITTED, TASK_STATUS_SUBMIT_FAILED,
-    TASK_STATUS_SUBMIT_RETRYING, TASK_STATUS_RUNNING, TASK_STATUS_SUCCEEDED,
-    TASK_STATUS_FAILED, TASK_STATUS_RETRYING)
-
-
-# Suite status strings.
-SUITE_STATUS_HELD = "held"
-SUITE_STATUS_RUNNING = "running"
-SUITE_STATUS_STOPPING = "stopping"
-SUITE_STATUS_RUNNING_TO_STOP = "running to stop at %s"
-SUITE_STATUS_RUNNING_TO_HOLD = "running to hold at %s"
-# Regex to extract the stop or hold point.
-SUITE_STATUS_SPLIT_REC = re.compile('^([a-z ]+ at )(.*)$')
-
-# Pseudo status strings for use by suite monitors.
-#   Use before attempting to determine status:
-SUITE_STATUS_NOT_CONNECTED = "not connected"
-#   Use prior to first status update:
-SUITE_STATUS_CONNECTED = "connected"
-SUITE_STATUS_INITIALISING = "initialising"
-#   Use when the suite is not running:
-SUITE_STATUS_STOPPED = "stopped"
-SUITE_STATUS_STOPPED_WITH = "stopped with '%s'"
-
-
-def get_suite_status_string(paused, stopping, will_pause_at, will_stop_at):
-    """Construct a suite status summary string for client programs.
-
-    This is in a function for re-use in monitor and GUI back-compat code
-    (clients at cylc version <= 6.9.1 construct their own status string).
-
-    """
-    if paused:
-        return SUITE_STATUS_HELD
-    elif stopping:
-        return SUITE_STATUS_STOPPING
-    elif will_pause_at:
-        return SUITE_STATUS_RUNNING_TO_HOLD % will_pause_at
-    elif will_stop_at:
-        return SUITE_STATUS_RUNNING_TO_STOP % will_stop_at
-    else:
-        return SUITE_STATUS_RUNNING
-
-
-class SuiteStillInitialisingError(Exception):
-    """Exception raised if a summary is requested before the first update.
-
-    This can happen if client connects during start-up for large suites.
-
-    """
-    def __str__(self):
-        return "Suite initializing..."
-
-
-class StateSummaryServer(PyroServer):
-    """Server-side suite state summary interface."""
-
-    _INSTANCE = None
-
-    @classmethod
-    def get_inst(cls, run_mode=None):
-        """Return a singleton instance."""
-        if cls._INSTANCE is None:
-            cls._INSTANCE = cls(run_mode)
-        return cls._INSTANCE
-
-    def __init__(self, run_mode):
-        super(StateSummaryServer, self).__init__()
-        self.task_summary = {}
-        self.global_summary = {}
-        self.family_summary = {}
-        self.run_mode = run_mode
-        self.first_update_completed = False
-        self._summary_update_time = None
-
-        self.state_count_totals = {}
-        self.state_count_cycles = {}
-
-    def update(self, tasks, tasks_rh, min_point, max_point, max_point_rh,
-               paused, will_pause_at, stopping, will_stop_at, ns_defn_order,
-               reloading):
-        global_summary = {}
-        family_summary = {}
-
-        task_summary, task_states = self._get_tasks_info(tasks, tasks_rh)
-
-        fam_states = {}
-        all_states = []
-        config = SuiteConfig.get_inst()
-        ancestors_dict = config.get_first_parent_ancestors()
-
-        # Compute state_counts (total, and per cycle).
-        state_count_totals = {}
-        state_count_cycles = {}
-
-        for point_string, c_task_states in task_states:
-            # For each cycle point, construct a family state tree
-            # based on the first-parent single-inheritance tree
-
-            c_fam_task_states = {}
-
-            count = {}
-
-            for key in c_task_states:
-                state = c_task_states[key]
-                if state is None:
-                    continue
-                try:
-                    count[state] += 1
-                except KeyError:
-                    count[state] = 1
-
-                all_states.append(state)
-                for parent in ancestors_dict.get(key, []):
-                    if parent == key:
-                        continue
-                    c_fam_task_states.setdefault(parent, set([]))
-                    c_fam_task_states[parent].add(state)
-
-            state_count_cycles[point_string] = count
-
-            for fam, child_states in c_fam_task_states.items():
-                f_id = TaskID.get(fam, point_string)
-                state = extract_group_state(child_states)
-                if state is None:
-                    continue
-                try:
-                    famcfg = config.cfg['runtime'][fam]
-                except KeyError:
-                    famcfg = {}
-                description = famcfg.get('description')
-                title = famcfg.get('title')
-                family_summary[f_id] = {'name': fam,
-                                        'description': description,
-                                        'title': title,
-                                        'label': point_string,
-                                        'state': state}
-
-        state_count_totals = {}
-        for point_string, count in state_count_cycles.items():
-            for state, state_count in count.items():
-                state_count_totals.setdefault(state, 0)
-                state_count_totals[state] += state_count
-
-        all_states.sort()
-
-        global_summary['oldest cycle point string'] = (
-            self.str_or_None(min_point))
-        global_summary['newest cycle point string'] = (
-            self.str_or_None(max_point))
-        global_summary['newest runahead cycle point string'] = (
-            self.str_or_None(max_point_rh))
-        if cylc.flags.utc:
-            global_summary['daemon time zone info'] = TIME_ZONE_UTC_INFO
-        else:
-            global_summary['daemon time zone info'] = TIME_ZONE_LOCAL_INFO
-        global_summary['last_updated'] = time.time()
-        global_summary['run_mode'] = self.run_mode
-        global_summary['states'] = all_states
-        global_summary['namespace definition order'] = ns_defn_order
-        global_summary['reloading'] = reloading
-        global_summary['state totals'] = state_count_totals
-
-        # Construct a suite status string for use by monitoring clients.
-        global_summary['status_string'] = get_suite_status_string(
-            paused, stopping, will_pause_at, will_stop_at)
-
-        # TODO - delete this block post back-compat concerns (<= 6.9.1):
-        #  Report separate status string components for older clients that
-        # construct their own suite status strings.
-        global_summary['paused'] = paused
-        global_summary['stopping'] = stopping
-        global_summary['will_pause_at'] = will_pause_at
-        global_summary['will_stop_at'] = will_stop_at
-
-        self._summary_update_time = time.time()
-
-        # Replace the originals (atomic update, for access from other threads).
-        self.task_summary = task_summary
-        self.global_summary = global_summary
-        self.family_summary = family_summary
-        self.first_update_completed = True
-        self.state_count_totals = state_count_totals
-        self.state_count_cycles = state_count_cycles
-
-    def _get_tasks_info(self, tasks, tasks_rh):
-        """Retrieve task summary info and states."""
-
-        task_summary = {}
-        task_states = {}
-
-        for task in tasks:
-            ts = task.get_state_summary()
-            task_summary[task.identity] = ts
-            name, point_string = TaskID.split(task.identity)
-            task_states.setdefault(point_string, {})
-            task_states[point_string][name] = ts['state']
-
-        for task in tasks_rh:
-            ts = task.get_state_summary()
-            ts['state'] = TASK_STATUS_RUNAHEAD
-            task_summary[task.identity] = ts
-            name, point_string = TaskID.split(task.identity)
-            task_states.setdefault(point_string, {})
-            task_states[point_string][name] = TASK_STATUS_RUNAHEAD
-
-        return task_summary, task_states.items()
-
-    def str_or_None(self, s):
-        if s:
-            return str(s)
-        else:
-            return None
-
-    def get_state_totals(self):
-        # (Access to this is controlled via the suite_identity server.)
-        return (self.state_count_totals, self.state_count_cycles)
-
-    def get_state_summary(self):
-        """Return the global, task, and family summary data structures."""
-        check_access_priv(self, 'full-read')
-        self.report('get_state_summary')
-        if not self.first_update_completed:
-            raise SuiteStillInitialisingError()
-        return (self.global_summary, self.task_summary, self.family_summary)
-
-    def get_summary_update_time(self):
-        """Return the last time the summaries were changed (Unix time)."""
-        check_access_priv(self, 'state-totals')
-        self.report('get_state_summary_update_time')
-        if not self.first_update_completed:
-            raise SuiteStillInitialisingError()
-        return self._summary_update_time
-
-
-class StateSummaryClient(PyroClient):
-    """Client-side suite state summary interface."""
-
-    target_server_object = PYRO_STATE_OBJ_NAME
-
-    def get_suite_state_summary(self):
-        return self.call_server_func("get_state_summary")
-
-    def get_suite_state_summary_update_time(self):
-        return self.call_server_func("get_summary_update_time")
-
-
-def extract_group_state(child_states, is_stopped=False):
-    """Summarise child states as a group."""
-
-    ordered_states = [TASK_STATUS_SUBMIT_FAILED, TASK_STATUS_FAILED,
-                      TASK_STATUS_EXPIRED, TASK_STATUS_SUBMIT_RETRYING,
-                      TASK_STATUS_RETRYING, TASK_STATUS_RUNNING,
-                      TASK_STATUS_SUBMITTED, TASK_STATUS_READY,
-                      TASK_STATUS_QUEUED, TASK_STATUS_WAITING,
-                      TASK_STATUS_HELD, TASK_STATUS_SUCCEEDED,
-                      TASK_STATUS_RUNAHEAD]
-    if is_stopped:
-        ordered_states = [TASK_STATUS_SUBMIT_FAILED, TASK_STATUS_FAILED,
-                          TASK_STATUS_RUNNING, TASK_STATUS_SUBMITTED,
-                          TASK_STATUS_EXPIRED, TASK_STATUS_READY,
-                          TASK_STATUS_SUBMIT_RETRYING, TASK_STATUS_RETRYING,
-                          TASK_STATUS_SUCCEEDED, TASK_STATUS_QUEUED,
-                          TASK_STATUS_WAITING, TASK_STATUS_HELD,
-                          TASK_STATUS_RUNAHEAD]
-    for state in ordered_states:
-        if state in child_states:
-            return state
-    return None
-
-
-def get_id_summary(id_, task_state_summary, fam_state_summary, id_family_map):
-    """Return some state information about a task or family id."""
-    prefix_text = ""
-    meta_text = ""
-    sub_text = ""
-    sub_states = {}
-    stack = [(id_, 0)]
-    done_ids = []
-    for summary in [task_state_summary, fam_state_summary]:
-        if id_ in summary:
-            title = summary[id_].get('title')
-            if title:
-                meta_text += "\n" + title.strip()
-            description = summary[id_].get('description')
-            if description:
-                meta_text += "\n" + description.strip()
-    while stack:
-        this_id, depth = stack.pop(0)
-        if this_id in done_ids:  # family dive down will give duplicates
-            continue
-        done_ids.append(this_id)
-        prefix = "\n" + " " * 4 * depth + this_id
-        if this_id in task_state_summary:
-            submit_num = task_state_summary[this_id].get('submit_num')
-            if submit_num:
-                prefix += "(%02d)" % submit_num
-            state = task_state_summary[this_id]['state']
-            sub_text += prefix + " " + state
-            sub_states.setdefault(state, 0)
-            sub_states[state] += 1
-        elif this_id in fam_state_summary:
-            name, point_string = TaskID.split(this_id)
-            sub_text += prefix + " " + fam_state_summary[this_id]['state']
-            for child in reversed(sorted(id_family_map[name])):
-                child_id = TaskID.get(child, point_string)
-                stack.insert(0, (child_id, depth + 1))
-        if not prefix_text:
-            prefix_text = sub_text.strip()
-            sub_text = ""
-    if len(sub_text.splitlines()) > 10:
-        state_items = sub_states.items()
-        state_items.sort()
-        state_items.sort(lambda x, y: cmp(y[1], x[1]))
-        sub_text = ""
-        for state, number in state_items:
-            sub_text += "\n    {0} tasks {1}".format(number, state)
-    if sub_text and meta_text:
-        sub_text = "\n" + sub_text
-    text = prefix_text + meta_text + sub_text
-    if not text:
-        return id_
-    return text
+"""Wrap communications daemon for a suite."""
+
+from cylc.network.method import METHOD
+
+if METHOD == "https":
+    from cylc.network.https.suite_state import (
+        StateSummaryServer, StateSummaryClient, extract_group_state,
+        get_id_summary, SUITE_STATUS_SPLIT_REC, get_suite_status_string,
+        StateSummaryClient, SuiteStillInitialisingError,
+        SUITE_STATUS_NOT_CONNECTED, SUITE_STATUS_CONNECTED,
+        SUITE_STATUS_INITIALISING, SUITE_STATUS_STOPPED, SUITE_STATUS_STOPPING,
+        SUITE_STATUS_STOPPED_WITH
+    )
diff --git a/lib/cylc/network/task_msgqueue.py b/lib/cylc/network/task_msgqueue.py
index b485479bb69..c89cea24952 100644
--- a/lib/cylc/network/task_msgqueue.py
+++ b/lib/cylc/network/task_msgqueue.py
@@ -15,41 +15,12 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see .
+"""Wrap communications daemon for a suite."""
 
-from Queue import Queue
-from cylc.owner import USER
-from cylc.suite_host import get_hostname
-from cylc.network.pyro_base import PyroClient, PyroServer
-from cylc.network import check_access_priv
+from cylc.network.method import METHOD
 
 
-class TaskMessageServer(PyroServer):
-    """Server-side task messaging interface"""
-
-    def __init__(self):
-        super(TaskMessageServer, self).__init__()
-        self.queue = Queue()
-
-    def put(self, task_id, priority, message):
-        check_access_priv(self, 'full-control')
-        self.report('task_message')
-        self.queue.put((task_id, priority, message))
-        return (True, 'Message queued')
-
-    def get_queue(self):
-        return self.queue
-
-
-class TaskMessageClient(PyroClient):
-    """Client-side task messaging interface"""
-
-    def __init__(self, suite, task_id, owner=USER,
-                 host=get_hostname(), pyro_timeout=None, port=None):
-        self.target_server_object = "task_pool"
-        self.task_id = task_id
-        super(TaskMessageClient, self).__init__(
-            suite, owner, host, pyro_timeout, port)
-
-    def put(self, *args):
-        args = [self.task_id] + list(args)
-        self.call_server_func('put', *args)
+if METHOD == "https":
+    from cylc.network.https.task_msgqueue import (
+        TaskMessageServer, TaskMessageClient
+    )
diff --git a/lib/cylc/registration.py b/lib/cylc/registration.py
index ccd35829906..e414d4a4da3 100644
--- a/lib/cylc/registration.py
+++ b/lib/cylc/registration.py
@@ -17,14 +17,17 @@
 # along with this program.  If not, see .
 """Simple suite name registration database."""
 
+import binascii
 import os
 import random
 import re
 import shlex
+import shutil
+import stat
 from string import ascii_letters, digits
-from subprocess import Popen, PIPE
+from subprocess import Popen, PIPE, check_call
 import sys
-from tempfile import NamedTemporaryFile
+from tempfile import NamedTemporaryFile, mkdtemp
 import traceback
 
 from cylc.cfgspec.globalcfg import GLOBAL_CFG
@@ -56,6 +59,8 @@ class RegistrationDB(object):
     PASSPHRASE_FILE_BASE = 'passphrase'
     PASSPHRASE_CHARSET = ascii_letters + digits
     PASSPHRASE_LEN = 20
+    SSL_CERTIFICATE_FILE_BASE = 'ssl.cert'
+    SSL_PRIVATE_KEY_FILE_BASE = 'ssl.pem'
 
     def __init__(self, dbpath=None):
         self.dbpath = dbpath or REGDB_PATH
@@ -119,6 +124,33 @@ def _dump_passphrase_to_dir(self, path, passphrase=None):
         if cylc.flags.verbose:
             print 'Generated suite passphrase: %s' % passphrase_file_name
 
+    def _dump_certificate_and_key_to_dir(self, path, passphrase=None):
+        """Dump SSL certificate to "ssl.cert" file in "path"."""
+        mkdir_p(path)
+
+        # Work in a user-read-write-only directory for guaranteed safety.
+        work_dir = mkdtemp()
+        source_pkey = os.path.join(work_dir, self.SSL_PRIVATE_KEY_FILE_BASE)
+        source_cert = os.path.join(work_dir, self.SSL_CERTIFICATE_FILE_BASE)
+        cn = '*'
+        altnames = 'DNS:*'
+        check_call(
+            ['openssl', 'req', '-nodes', '-batch', '-new', '-x509',
+             '-sha512', '-subj', '/O=Cylc suite' +
+             '/CN=%s/subjectAltName=%s' % (cn, altnames),
+             '-days', "3653",
+             '-keyout', source_pkey, '-out', source_cert])
+        os.chmod(source_pkey, stat.S_IRUSR)
+        os.chmod(source_cert, stat.S_IRUSR)
+        dest_pkey = os.path.join(path, self.SSL_PRIVATE_KEY_FILE_BASE)
+        dest_cert = os.path.join(path, self.SSL_CERTIFICATE_FILE_BASE)
+        shutil.copy(source_pkey, dest_pkey)
+        shutil.copy(source_cert, dest_cert)
+        shutil.rmtree(work_dir)
+        if cylc.flags.verbose:
+            print 'Generated suite SSL certificate: %s' % dest_cert
+            print 'Generated suite SSL private key: %s' % dest_pkey
+
     def dump_suite_data(self, suite, data):
         """Dump suite path and title in text file."""
         with open(os.path.join(self.dbpath, suite), 'w') as handle:
@@ -162,6 +194,67 @@ def load_all_passphrases(self):
 
         return self.local_passphrases
 
+    def load_item(self, suite, owner, host, item="certificate",
+                  create_ok=False):
+        """Search for an SSL certificate file for this suite or create it."""
+        suite_host = os.getenv('CYLC_SUITE_HOST')
+        suite_owner = os.getenv('CYLC_SUITE_OWNER')
+        env_keys = []
+        if suite == os.getenv('CYLC_SUITE_NAME'):
+            if is_remote_host(suite_host) or is_remote_user(suite_owner):
+                # 2(i)/ Task messaging call on a remote account.
+                # First look in the remote suite run directory than suite
+                # definition directory ($CYLC_SUITE_DEF_PATH is modified
+                # for remote tasks):
+                env_keys = ['CYLC_SUITE_RUN_DIR', 'CYLC_SUITE_DEF_PATH']
+            elif suite_host or suite_owner:
+                # 2(ii)/ Task messaging call on the suite host account.
+
+                # Could be a local task or a remote task with 'ssh
+                # messaging = True'. In either case use
+                # $CYLC_SUITE_DEF_PATH_ON_SUITE_HOST which never
+                # changes, not $CYLC_SUITE_DEF_PATH which gets
+                # modified for remote tasks as described above.
+                env_keys = ['CYLC_SUITE_DEF_PATH_ON_SUITE_HOST']
+        paths = []
+        for key in env_keys:
+            if key in os.environ:
+                paths.append(os.environ[key])
+        if suite == os.environ.get('CYLC_SUITE_NAME'):
+            paths.append(self.get_suitedir(suite))
+        else:
+            paths.insert(0, self.get_suitedir(suite))
+
+        for path in paths:
+            try:
+                return getattr(
+                    self, "load_%s_from_dir" % item)(path)
+            except (IOError, PassphraseError):
+                pass
+        if create_ok:
+            return self._dump_certificate_and_key_to_dir(path)
+        raise PassphraseError("Couldn't get %s" % item)
+
+    def load_certificate_from_dir(self, path):
+        file_name = os.path.join(path, self.SSL_CERTIFICATE_FILE_BASE)
+        try:
+            content = open(file_name).read()
+        except IOError:
+            raise
+        if not content:
+            raise PassphraseError("no content in %s" % file_name)
+        return file_name
+
+    def load_private_key_from_dir(self, path):
+        file_name = os.path.join(path, self.SSL_PRIVATE_KEY_FILE_BASE)
+        try:
+            content = open(file_name).read()
+        except IOError:
+            raise
+        if not content:
+            raise PassphraseError("no content in %s" % file_name)
+        return file_name
+
     def load_passphrase(self, suite, owner, host, cache_ok=True):
         """Search for passphrase file for suite, load and return content.
 
@@ -380,6 +473,18 @@ def register(self, name, path):
         except (IOError, PassphraseError):
             self._dump_passphrase_to_dir(path)
 
+        # Create a new private key for the suite if necessary
+        try:
+            self.load_private_key_from_dir(path)
+        except (IOError, PassphraseError):
+            self._dump_certificate_and_key_to_dir(path)
+        else:
+            # Create a new certificate for the suite if necessary
+            try:
+                self.load_certificate_from_dir(path)
+            except (IOError, PassphraseError):
+                self._dump_certificate_and_key_to_dir(path)
+
     def get_suite_data(self, suite):
         """Return {"path": path, "title": title} a suite."""
         suite = RegPath(suite).get()
@@ -441,7 +546,7 @@ def unregister(self, exp):
         """Un-register a suite."""
         unregistered_set = set()
         skipped_set = set()
-        ports_d = GLOBAL_CFG.get(['pyro', 'ports directory'])
+        ports_d = GLOBAL_CFG.get(['communication', 'ports directory'])
         for name in sorted(self.list_all_suites()):
             if not re.match(exp + r'\Z', name):
                 continue
@@ -454,7 +559,9 @@ def unregister(self, exp):
                 print >> sys.stderr, (
                     'SKIP UNREGISTER %s: port file exists' % (name))
                 continue
-            for base_name in ['passphrase', 'suite.rc.processed']:
+            for base_name in ['passphrase', 'suite.rc.processed',
+                              self.SSL_CERTIFICATE_FILE_BASE,
+                              self.SSL_PRIVATE_KEY_FILE_BASE]:
                 try:
                     os.unlink(os.path.join(data['path'], base_name))
                 except OSError:
diff --git a/lib/cylc/scheduler.py b/lib/cylc/scheduler.py
index 61669b6c4a1..d07e22985e2 100644
--- a/lib/cylc/scheduler.py
+++ b/lib/cylc/scheduler.py
@@ -53,7 +53,7 @@
     PYRO_CMD_OBJ_NAME, PYRO_BCAST_OBJ_NAME, PYRO_EXT_TRIG_OBJ_NAME,
     PYRO_INFO_OBJ_NAME, PYRO_LOG_OBJ_NAME)
 from cylc.network.ext_trigger import ExtTriggerServer
-from cylc.network.pyro_daemon import PyroDaemon
+from cylc.network.daemon import CommsDaemon
 from cylc.network.suite_broadcast import BroadcastServer
 from cylc.network.suite_command import SuiteCommandServer
 from cylc.network.suite_identifier import SuiteIdServer
@@ -92,24 +92,6 @@ class SchedulerStop(CylcError):
     pass
 
 
-class PyroRequestHandler(threading.Thread):
-    """Pyro request handler."""
-
-    def __init__(self, pyro):
-        threading.Thread.__init__(self)
-        self.pyro = pyro
-        self.quit = False
-        self.log = logging.getLogger('main')
-        self.log.debug("request handling thread starting")
-
-    def run(self):
-        while True:
-            self.pyro.handle_requests(timeout=1)
-            if self.quit:
-                break
-        self.log.debug("request handling thread exiting")
-
-
 class Scheduler(object):
     """Cylc scheduler server."""
 
@@ -182,8 +164,7 @@ def __init__(self, is_restart, options, args):
         self.suite_state = None
         self.command_queue = None
         self.pool = None
-        self.request_handler = None
-        self.pyro = None
+        self.comms_daemon = None
         self.state_dumper = None
 
         self._profile_amounts = {}
@@ -221,9 +202,9 @@ def start(self):
                 pri_dao.close()
 
         try:
-            self._configure_pyro()
             if not self.options.no_detach and not cylc.flags.debug:
                 daemonize(self)
+            self.configure_comms_daemon()
             self.configure()
             if self.options.profile_mode:
                 import cProfile
@@ -403,10 +384,8 @@ def configure(self):
 
         self.pool = TaskPool(
             self.suite, self.pri_dao, self.pub_dao, self.final_point,
-            self.pyro, self.log, self.run_mode)
+            self.comms_daemon, self.log, self.run_mode)
         self.state_dumper.pool = self.pool
-        self.request_handler = PyroRequestHandler(self.pyro)
-        self.request_handler.start()
 
         self.old_user_at_host_set = set()
         self.log_memory("scheduler.py: before load_tasks")
@@ -886,20 +865,26 @@ def process_command_queue(self):
 
         while True:
             try:
-                name, args = queue.get(False)
+                name, args, kwargs = queue.get(False)
             except Empty:
                 break
             print '  +', name
-            cmdstr = name + '(' + ','.join([str(a) for a in args]) + ')'
+            cmdstr = name + '(' + ', '.join([str(a) for a in args])
+            cmdstr += (
+                ', '.join(
+                    [key + '=' + str(value) for key, value in kwargs.items()]
+                ) + ')'
+            )
             try:
-                n_warnings = getattr(self, "command_%s" % name)(*args)
+                n_warnings = getattr(self, "command_%s" % name)(
+                    *args, **kwargs)
             except SchedulerStop:
                 self.log.info('Command succeeded: ' + cmdstr)
                 raise
-            except Exception, x:
+            except Exception as exc:
                 # Don't let a bad command bring the suite down.
                 self.log.warning(traceback.format_exc())
-                self.log.warning(str(x))
+                self.log.warning(str(exc))
                 self.log.warning('Command failed: ' + cmdstr)
             else:
                 if n_warnings:
@@ -988,8 +973,10 @@ def info_get_first_parent_ancestors(self, pruned=False):
         # single-inheritance hierarchy based on first parents
         return deepcopy(self.config.get_first_parent_ancestors(pruned))
 
-    def info_get_graph_raw(self, cto, ctn, group_nodes, ungroup_nodes,
-                           ungroup_recursive, group_all, ungroup_all):
+    def info_get_graph_raw(self, cto, ctn, group_nodes=None,
+                           ungroup_nodes=None,
+                           ungroup_recursive=False, group_all=False,
+                           ungroup_all=False):
         rgraph = self.config.get_graph_raw(
             cto, ctn, group_nodes, ungroup_nodes, ungroup_recursive, group_all,
             ungroup_all)
@@ -1039,25 +1026,25 @@ def command_set_stop_after_task(self, task_id):
         if TaskID.is_valid_id(task_id):
             self.set_stop_task(task_id)
 
-    def command_release_task(self, items, compat=None, _=None):
+    def command_release_tasks(self, items):
         """Release tasks."""
-        return self.pool.release_tasks(items, compat)
+        return self.pool.release_tasks(items)
 
-    def command_poll_tasks(self, items, compat=None, _=None):
+    def command_poll_tasks(self, items):
         """Poll all tasks or a task/family if options are provided."""
-        return self.pool.poll_task_jobs(items, compat)
+        return self.pool.poll_task_jobs(items)
 
-    def command_kill_tasks(self, items, compat=None, _=False):
+    def command_kill_tasks(self, items):
         """Kill all tasks or a task/family if options are provided."""
-        return self.pool.kill_task_jobs(items, compat)
+        return self.pool.kill_task_jobs(items)
 
     def command_release_suite(self):
         """Release all task proxies in the suite."""
         self.release_suite()
 
-    def command_hold_task(self, items, compat=None, _=False):
+    def command_hold_tasks(self, items):
         """Hold selected task proxies in the suite."""
-        return self.pool.hold_tasks(items, compat)
+        return self.pool.hold_tasks(items)
 
     def command_hold_suite(self):
         """Hold all task proxies in the suite."""
@@ -1080,14 +1067,13 @@ def command_remove_cycle(self, point_string, spawn=False):
         """Remove tasks in a cycle."""
         return self.pool.remove_tasks(point_string + "/*", spawn)
 
-    def command_remove_task(self, items, compat=None, _=None, spawn=False):
+    def command_remove_tasks(self, items, spawn=False):
         """Remove tasks."""
-        return self.pool.remove_tasks(items, spawn, compat)
+        return self.pool.remove_tasks(items, spawn)
 
-    def command_insert_task(
-            self, items, compat=None, _=None, stop_point_string=None):
+    def command_insert_tasks(self, items, stop_point_string=None):
         """Insert tasks."""
-        return self.pool.insert_tasks(items, stop_point_string, compat)
+        return self.pool.insert_tasks(items, stop_point_string)
 
     def command_nudge(self):
         """Cause the task processing loop to be invoked"""
@@ -1114,8 +1100,8 @@ def command_reload_suite(self):
         self.state_dumper.set_cts(self.initial_point, self.final_point)
         self.do_update_state_summary = True
 
-    def command_set_runahead(self, *args):
-        self.pool.set_runahead(*args)
+    def command_set_runahead(self, interval=None):
+        self.pool.set_runahead(interval=interval)
 
     def set_suite_timer(self, reset=False):
         """Set suite's timeout timer."""
@@ -1128,6 +1114,49 @@ def set_suite_timer(self, reset=False):
                     self._get_events_conf(self.EVENT_TIMEOUT)),
                 get_current_time_string())
 
+    def configure_comms_daemon(self):
+        """Create and configure daemon."""
+        self.comms_daemon = CommsDaemon(self.suite, self.suite_dir)
+        self.port = self.comms_daemon.get_port()
+        self.port_file = os.path.join(
+            GLOBAL_CFG.get(['communication', 'ports directory']), self.suite)
+        try:
+            port, host = open(self.port_file).read().splitlines()
+        except (IOError, ValueError):
+            # Suite is not likely to be running if port file does not exist
+            # or if port file does not contain good values of port and host.
+            pass
+        else:
+            sys.stderr.write(
+                (
+                    r"""ERROR: port file exists: %(port_file)s
+
+If %(suite)s is not running, delete the port file and try again.  If it is
+running but not responsive, kill any left over suite processes too.
+
+To see if %(suite)s is running on '%(host)s:%(port)s':
+ * cylc scan -n '\b%(suite)s\b' %(host)s
+ * cylc ping -v --host=%(host)s %(suite)s
+ * ssh %(host)s "pgrep -a -P 1 -fu $USER 'cylc-r.* \b%(suite)s\b'"
+
+"""
+                ) % {
+                    "host": host,
+                    "port": port,
+                    "port_file": self.port_file,
+                    "suite": self.suite,
+                }
+            )
+            raise SchedulerError(
+                "ERROR, port file exists: %s" % self.port_file)
+        try:
+            with open(self.port_file, 'w') as handle:
+                handle.write("%d\n%s\n" % (self.port, self.host))
+        except IOError as exc:
+            sys.stderr.write(str(exc) + "\n")
+            raise SchedulerError(
+                'ERROR, cannot write port file: %s' % self.port_file)
+
     def load_suiterc(self, reconfigure):
         """Load and log the suite definition."""
 
@@ -1252,26 +1281,26 @@ def configure_suite(self, reconfigure=False):
             self.logfile = slog.get_path()
 
             suite_id = SuiteIdServer.get_inst(self.suite, self.owner)
-            self.pyro.connect(suite_id, PYRO_SUITEID_OBJ_NAME)
+            self.comms_daemon.connect(suite_id, PYRO_SUITEID_OBJ_NAME)
 
             bcast = BroadcastServer.get_inst(
                 self.config.get_linearized_ancestors())
-            self.pyro.connect(bcast, PYRO_BCAST_OBJ_NAME)
+            self.comms_daemon.connect(bcast, PYRO_BCAST_OBJ_NAME)
 
             self.command_queue = SuiteCommandServer()
-            self.pyro.connect(self.command_queue, PYRO_CMD_OBJ_NAME)
+            self.comms_daemon.connect(self.command_queue, PYRO_CMD_OBJ_NAME)
 
             ets = ExtTriggerServer.get_inst()
-            self.pyro.connect(ets, PYRO_EXT_TRIG_OBJ_NAME)
+            self.comms_daemon.connect(ets, PYRO_EXT_TRIG_OBJ_NAME)
 
             self.info_interface = SuiteInfoServer(self.info_commands)
-            self.pyro.connect(self.info_interface, PYRO_INFO_OBJ_NAME)
+            self.comms_daemon.connect(self.info_interface, PYRO_INFO_OBJ_NAME)
 
             self.log_interface = SuiteLogServer(slog)
-            self.pyro.connect(self.log_interface, PYRO_LOG_OBJ_NAME)
+            self.comms_daemon.connect(self.log_interface, PYRO_LOG_OBJ_NAME)
 
             self.suite_state = StateSummaryServer.get_inst(self.run_mode)
-            self.pyro.connect(self.suite_state, PYRO_STATE_OBJ_NAME)
+            self.comms_daemon.connect(self.suite_state, PYRO_STATE_OBJ_NAME)
 
     def configure_suite_environment(self):
         # static cylc and suite-specific variables:
@@ -1286,7 +1315,7 @@ def configure_suite_environment(self):
             'CYLC_SUITE_REG_NAME': self.suite,  # DEPRECATED
             'CYLC_SUITE_HOST': str(self.host),
             'CYLC_SUITE_OWNER': self.owner,
-            'CYLC_SUITE_PORT': str(self.pyro.get_port()),
+            'CYLC_SUITE_PORT': str(self.comms_daemon.get_port()),
             # DEPRECATED
             'CYLC_SUITE_REG_PATH': RegPath(self.suite).get_fpath(),
             'CYLC_SUITE_DEF_PATH_ON_SUITE_HOST': self.suite_dir,
@@ -1793,21 +1822,17 @@ def shutdown(self, reason=''):
                     # ignore errors here in order to shut down cleanly
                     self.log.warning('Final state dump failed: ' + str(exc))
 
-        if self.request_handler:
-            self.request_handler.quit = True
-            self.request_handler.join()
-
         for iface in [self.command_queue,
                       SuiteIdServer.get_inst(), StateSummaryServer.get_inst(),
                       ExtTriggerServer.get_inst(), BroadcastServer.get_inst()]:
             try:
-                self.pyro.disconnect(iface)
+                self.comms_daemon.disconnect(iface)
             except KeyError:
                 # Wasn't connected yet.
                 pass
 
-        if self.pyro:
-            self.pyro.shutdown()
+        if self.comms_daemon:
+            self.comms_daemon.shutdown()
 
         try:
             os.unlink(self.port_file)
@@ -1896,21 +1921,22 @@ def paused(self):
     def will_pause_at(self):
         return self.pool.get_hold_point()
 
-    def command_trigger_task(self, items, compat=None, _=None):
+    def command_trigger_tasks(self, items):
         """Trigger tasks."""
-        return self.pool.trigger_tasks(items, compat)
+        print "Trigger", items
+        return self.pool.trigger_tasks(items)
 
-    def command_dry_run_task(self, items, compat=None):
-        """Dry-run a task, e.g. edit run."""
-        return self.pool.dry_run_task(items, compat)
+    def command_dry_run_tasks(self, items):
+        """Dry-run tasks, e.g. edit run."""
+        return self.pool.dry_run_task(items)
 
-    def command_reset_task_state(self, items, compat=None, state=None, _=None):
+    def command_reset_task_states(self, items, state=None):
         """Reset the state of tasks."""
-        return self.pool.reset_task_states(items, state, compat)
+        return self.pool.reset_task_states(items, state)
 
-    def command_spawn_tasks(self, items, compat=None, _=None):
+    def command_spawn_tasks(self, items):
         """Force spawn task successors."""
-        return self.pool.spawn_tasks(items, compat)
+        return self.pool.spawn_tasks(items)
 
     def filter_initial_task_list(self, inlist):
         included_by_rc = self.config.cfg[
diff --git a/lib/cylc/task_pool.py b/lib/cylc/task_pool.py
index f8d5e4f3edd..28f0b8b2bdf 100644
--- a/lib/cylc/task_pool.py
+++ b/lib/cylc/task_pool.py
@@ -92,9 +92,9 @@ def __init__(self, suite, pri_dao, pub_dao, stop_point, pyro, log,
             config.get_max_num_active_cycle_points())
         self._prev_runahead_base_point = None
         self._prev_runahead_sequence_points = None
-        self.message_queue = TaskMessageServer()
+        self.message_queue = TaskMessageServer(self.suite_name)
 
-        self.pyro.connect(self.message_queue, "task_pool")
+        self.pyro.connect(self.message_queue, "message")
 
         self.pool = {}
         self.runahead_pool = {}
@@ -125,11 +125,9 @@ def assign_queues(self):
             for taskname in qconfig[queue]['members']:
                 self.myq[taskname] = queue
 
-    def insert_tasks(self, items, stop_point_str, compat=None):
+    def insert_tasks(self, items, stop_point_str):
         """Insert tasks."""
         n_warnings = 0
-        if isinstance(items, str) or compat is not None:
-            items = [items + "." + compat]
         config = SuiteConfig.get_inst()
         names = config.get_task_name_list()
         fams = config.runtime['first-parent descendants']
@@ -814,7 +812,7 @@ def report_stalled_task_deps(self):
             for unsatisfied in prereqs['prereqs']:
                 self.log.warning(" * %s" % unsatisfied)
 
-    def poll_task_jobs(self, items=None, compat=None):
+    def poll_task_jobs(self, items=None):
         """Poll jobs of active tasks.
 
         If items is specified, poll active tasks matching given IDs.
@@ -822,7 +820,7 @@ def poll_task_jobs(self, items=None, compat=None):
         """
         if self.run_mode == 'simulation':
             return
-        itasks, n_warnings = self._filter_task_proxies(items, compat)
+        itasks, n_warnings = self._filter_task_proxies(items)
         active_itasks = []
         for itask in itasks:
             if itask.state.status in TASK_STATUSES_POLLABLE:
@@ -846,13 +844,13 @@ def poll_task_jobs_callback(self, ctx):
             },
         )
 
-    def kill_task_jobs(self, items=None, compat=None):
+    def kill_task_jobs(self, items=None):
         """Kill jobs of active tasks.
 
         If items is specified, kill active tasks matching given IDs.
 
         """
-        itasks, n_warnings = self._filter_task_proxies(items, compat)
+        itasks, n_warnings = self._filter_task_proxies(items)
         active_itasks = []
         for itask in itasks:
             is_active = itask.state.status in TASK_STATUSES_KILLABLE
@@ -936,16 +934,16 @@ def set_hold_point(self, point):
                 if itask.point > point:
                     itask.state.reset_state(TASK_STATUS_HELD)
 
-    def hold_tasks(self, items, compat=None):
+    def hold_tasks(self, items):
         """Hold tasks with IDs matching any item in "ids"."""
-        itasks, n_warnings = self._filter_task_proxies(items, compat)
+        itasks, n_warnings = self._filter_task_proxies(items)
         for itask in itasks:
             itask.state.reset_state(TASK_STATUS_HELD)
         return n_warnings
 
-    def release_tasks(self, items, compat=None):
+    def release_tasks(self, items):
         """Release held tasks with IDs matching any item in "ids"."""
-        itasks, n_warnings = self._filter_task_proxies(items, compat)
+        itasks, n_warnings = self._filter_task_proxies(items)
         for itask in itasks:
             itask.state.release()
         return n_warnings
@@ -1357,20 +1355,20 @@ def remove_spent_tasks(self):
             self.remove(itask)
         return len(spent)
 
-    def spawn_tasks(self, items, compat):
+    def spawn_tasks(self, items):
         """Force tasks to spawn successors if they haven't already.
 
         """
-        itasks, n_warnings = self._filter_task_proxies(items, compat)
+        itasks, n_warnings = self._filter_task_proxies(items)
         for itask in itasks:
             if not itask.has_spawned:
                 itask.log(INFO, "forced spawning")
                 self.force_spawn(itask)
         return n_warnings
 
-    def reset_task_states(self, items, status, compat):
+    def reset_task_states(self, items, status):
         """Reset task states."""
-        itasks, n_warnings = self._filter_task_proxies(items, compat)
+        itasks, n_warnings = self._filter_task_proxies(items)
         for itask in itasks:
             itask.log(INFO, "resetting state to %s" % status)
             if status == TASK_STATUS_READY:
@@ -1383,18 +1381,18 @@ def reset_task_states(self, items, status, compat):
                 itask.state.reset_state(status)
         return n_warnings
 
-    def remove_tasks(self, items, spawn=False, compat=None):
+    def remove_tasks(self, items, spawn=False):
         """Remove tasks from pool."""
-        itasks, n_warnings = self._filter_task_proxies(items, compat)
+        itasks, n_warnings = self._filter_task_proxies(items)
         for itask in itasks:
             if spawn:
                 self.force_spawn(itask)
             self.remove(itask, 'by request')
         return n_warnings
 
-    def trigger_tasks(self, items, compat=None):
+    def trigger_tasks(self, items):
         """Trigger tasks."""
-        itasks, n_warnings = self._filter_task_proxies(items, compat)
+        itasks, n_warnings = self._filter_task_proxies(items)
         for itask in itasks:
             if itask.state.status in TASK_STATUSES_ACTIVE:
                 self.log.warning('%s: already triggered' % itask.identity)
@@ -1403,16 +1401,13 @@ def trigger_tasks(self, items, compat=None):
             itask.manual_trigger = True
             if not itask.state.status == TASK_STATUS_QUEUED:
                 itask.state.reset_state(TASK_STATUS_READY)
+        cylc.flags.pflag = True
         return n_warnings
 
-    def dry_run_task(self, items, compat=None):
+    def dry_run_task(self, items):
         """Create job file for "cylc trigger --edit"."""
-        itasks, n_warnings = self._filter_task_proxies(items, compat)
+        itasks, n_warnings = self._filter_task_proxies(items)
         if len(itasks) > 1:
-            if isinstance(items, str) and compat is not None:
-                items = items + "." + compat
-            elif compat is not None:
-                items = "*." + compat
             self.log.warning("Unique task match not found: %s" % items)
             n_warnings += 1
         else:
@@ -1487,7 +1482,6 @@ def sim_time_check(self):
     def shutdown(self):
         if not self.no_active_tasks():
             self.log.warning("some active tasks will be orphaned")
-        self.pyro.disconnect(self.message_queue)
 
     def waiting_tasks_ready(self):
         """Waiting tasks can become ready for internal reasons.
@@ -1577,7 +1571,7 @@ def match_ext_triggers(self):
             if itask.state.external_triggers:
                 ets.retrieve(itask)
 
-    def _filter_task_proxies(self, items, compat=None):
+    def _filter_task_proxies(self, items):
         """Return task proxies that match names, points, states in items.
 
         In the new form, the arguments should look like:
@@ -1585,41 +1579,12 @@ def _filter_task_proxies(self, items, compat=None):
                  the general form name[.point][:state] or [point/]name[:state]
                  where name is a glob-like pattern for matching a task name or
                  a family name.
-        compat -- not used
-
-        In the old form, "items" is a string containing a regular expression
-        for matching task/family names, and "compat" is a string containing a
-        point string.
 
         """
         itasks = []
         n_warnings = 0
-        if not items and compat is None:
+        if not items:
             itasks += self.get_all_tasks()
-        elif isinstance(items, str) or compat is not None:
-            try:
-                point_str = standardise_point_string(compat)
-            except ValueError as exc:
-                self.log.warning(
-                    self.ERR_PREFIX_TASKID_MATCH +
-                    ("%s.%s: %s" % (items, compat, exc)))
-                n_warnings += 1
-            else:
-                name_rec = re.compile(items)
-                for itask in self.get_all_tasks():
-                    nss = itask.tdef.namespace_hierarchy
-                    if (
-                            (point_str is None or
-                             str(itask.point) == point_str) and
-                            (name_rec.match(itask.tdef.name) or
-                             any([name_rec.match(ns) for ns in nss]))
-                    ):
-                        itasks.append(itask)
-                if not itasks:
-                    self.log.warning(
-                        self.ERR_PREFIX_TASKID_MATCH +
-                        ("%s.%s" % (items, compat)))
-                    n_warnings += 1
         else:
             for item in items:
                 point_str, name_str, status = self._parse_task_item(item)
diff --git a/tests/authentication/00-identity.t b/tests/authentication/00-identity.t
index 1e23a44a5f7..e51bfacde33 100644
--- a/tests/authentication/00-identity.t
+++ b/tests/authentication/00-identity.t
@@ -52,13 +52,13 @@ __END__
 TEST_NAME="${TEST_NAME_BASE}-show"
 run_fail "${TEST_NAME}" cylc show "${SUITE_NAME}"
 cylc log "${SUITE_NAME}" > suite.log1
-grep_ok "\[client-connect] DENIED (privilege 'identity' < 'description') ${USER}@.*:cylc-show" suite.log1
+grep_ok "\[client-connect\] DENIED (privilege 'identity' < 'description') ${USER}@.*:cylc-show" suite.log1
 
 # Commands should be denied.
 TEST_NAME="${TEST_NAME_BASE}-stop"
 run_fail "${TEST_NAME}" cylc stop "${SUITE_NAME}"
 cylc log "${SUITE_NAME}" > suite.log2
-grep_ok "\[client-connect] DENIED (privilege 'identity' < 'shutdown') ${USER}@.*:cylc-stop" suite.log2
+grep_ok "\[client-connect\] DENIED (privilege 'identity' < 'shutdown') ${USER}@.*:cylc-stop" suite.log2
 
 # Restore the passphrase.
 mv "${TEST_DIR}/${SUITE_NAME}/passphrase.DIS" \
diff --git a/tests/authentication/05-full-control.t b/tests/authentication/05-full-control.t
index 64d5732a95e..e8cf367d3bf 100644
--- a/tests/authentication/05-full-control.t
+++ b/tests/authentication/05-full-control.t
@@ -55,23 +55,23 @@ __END__
 TEST_NAME="${TEST_NAME_BASE}-show1"
 run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}"
 cylc log "${SUITE_NAME}" > suite.log1
-grep_ok "\[client-command] get_suite_info ${USER}@.*:cylc-show" suite.log1
+grep_ok "\[client-command\] get_suite_info ${USER}@.*:cylc-show" suite.log1
 
 # "cylc show" (task info) OK.
 TEST_NAME="${TEST_NAME_BASE}-show2"
 run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" foo.1
 cylc log "${SUITE_NAME}" > suite.log2
-grep_ok "\[client-command] get_task_info ${USER}@.*:cylc-show" suite.log2
+grep_ok "\[client-command\] get_task_info ${USER}@.*:cylc-show" suite.log2
 
 # Commands OK.
 # (Reset to same state).
 TEST_NAME="${TEST_NAME_BASE}-trigger"
 run_ok "${TEST_NAME}" cylc reset "${SUITE_NAME}" -s failed foo 1
 cylc log "${SUITE_NAME}" > suite.log3
-grep_ok "\[client-command] reset_task_state ${USER}@.*:cylc-reset" suite.log3
+grep_ok "\[client-command\] reset_task_states ${USER}@.*:cylc-reset" suite.log3
 
 # Shutdown and purge.
 TEST_NAME="${TEST_NAME_BASE}-stop"
-run_ok "${TEST_NAME}" cylc stop --max-polls=10 --interval=1 "${SUITE_NAME}"
+run_ok "${TEST_NAME}" cylc stop --max-polls=20 --interval=1 "${SUITE_NAME}"
 purge_suite "${SUITE_NAME}"
 exit
diff --git a/tests/authentication/07-back-compat.t b/tests/authentication/07-back-compat.t
deleted file mode 100644
index 4ce6f24fc3c..00000000000
--- a/tests/authentication/07-back-compat.t
+++ /dev/null
@@ -1,172 +0,0 @@
-#!/bin/bash
-# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
-# Copyright (C) 2008-2016 NIWA
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
-
-# Test authentication - ignore old client denials, report bad new clients.
-
-. $(dirname $0)/test_header
-set_test_number 23
-
-# Set things up and run the suite.
-# Choose the default global.rc hash settings, for reference.
-create_test_globalrc '' '
-[authentication]
-    hashes = sha256,md5
-    scan hash = md5'
-install_suite "${TEST_NAME_BASE}" basic
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-cylc run "${SUITE_NAME}"
-
-# Scan to grab the suite's port.
-sleep 5  # Wait for the suite to initialize.
-PORT=$(cylc ping -v "${SUITE_NAME}" | cut -d':' -f 2)
-
-# Simulate an old client with the wrong passphrase.
-ERR_PATH="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/log/suite/err"
-TEST_NAME="${TEST_NAME_BASE}-old-client-snapshot-err"
-run_ok "${TEST_NAME}" cp "${ERR_PATH}" err-before-scan
-TEST_NAME="${TEST_NAME_BASE}-old-client-simulate"
-run_fail "${TEST_NAME}" python -c "
-import sys
-import Pyro.core
-uri = 'PYROLOC://localhost:' + sys.argv[1] + '/cylcid'
-proxy = Pyro.core.getProxyForURI(uri)
-proxy._setIdentification('0123456789abcdef')
-name, owner = proxy.id()" "${PORT}"
-grep_ok "ConnectionDeniedError" "${TEST_NAME}.stderr"
-
-# Check that the old client connection is not logged.
-# Get any new lines added to the error file.
-comm -13 err-before-scan "${ERR_PATH}" >"${TEST_NAME_BASE}-old-client-err-diff"
-TEST_NAME="${TEST_NAME_BASE}-log-old-client"
-run_fail "${TEST_NAME}" grep "WARNING - \[client-connect\] DENIED" \
-    "${TEST_NAME_BASE}-old-client-err-diff"
-
-# Simulate an old client with the right passphrase.
-ERR_PATH="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/log/suite/err"
-TEST_NAME="${TEST_NAME_BASE}-old-client-snapshot-ok"
-run_ok "${TEST_NAME}" cp "${ERR_PATH}" err-before-scan
-TEST_NAME="${TEST_NAME_BASE}-old-client-simulate-ok"
-PASSPHRASE=$(cat $(cylc get-dir $SUITE_NAME)/passphrase)
-run_ok "${TEST_NAME}" python -c "
-import sys
-import Pyro.core
-uri = 'PYROLOC://localhost:' + sys.argv[1] + '"/${USER}.${SUITE_NAME}.suite-info"'
-print >> sys.stderr, uri
-proxy = Pyro.core.getProxyForURI(uri)
-proxy._setIdentification('"$PASSPHRASE"')
-info = proxy.get('get_suite_info')" "${PORT}"
-grep_ok "\[client-command\] get_suite_info (user)@(host):(OLD_CLIENT) (uuid)" "$(cylc cat-log -l $SUITE_NAME)"
-
-# Simulate a new, suspicious client.
-TEST_NAME="${TEST_NAME_BASE}-new-bad-client-snapshot-err"
-run_ok "${TEST_NAME}" cp "${ERR_PATH}" err-before-scan
-TEST_NAME="${TEST_NAME_BASE}-new-bad-client-simulate"
-run_fail "${TEST_NAME}" python -c '
-import sys
-import Pyro.core, Pyro.protocol
-
-class MyConnValidator(Pyro.protocol.DefaultConnValidator):
-
-    """Create an incorrect but plausible auth token."""
-
-    def createAuthToken(self, authid, challenge, peeraddr, URI, daemon):
-        return "colonel_mustard:drawing_room:dagger:mystery:decea5ede57abbed"
-
-uri = "PYROLOC://localhost:" + sys.argv[1] + "/cylcid"
-proxy = Pyro.core.getProxyForURI(uri)
-proxy._setNewConnectionValidator(MyConnValidator())
-proxy._setIdentification("0123456789abcdef")
-proxy.identify()' "${PORT}"
-grep_ok "ConnectionDeniedError" "${TEST_NAME}.stderr"
-
-# Check that the new client connection failure is logged (it is suspicious).
-TEST_NAME="${TEST_NAME_BASE}-log-new-client"
-# Get any new lines added to the error file.
-comm -13 err-before-scan "${ERR_PATH}" >"${TEST_NAME}-new-client-err-diff"
-# Check the new lines for a connection denied report.
-grep_ok "WARNING - \[client-connect\] DENIED colonel_mustard@drawing_room:mystery dagger$" \
-    "${TEST_NAME}-new-client-err-diff"
-
-# Simulate a client with the wrong hash.
-TEST_NAME="${TEST_NAME_BASE}-new-wrong-hash-client-snapshot-err"
-run_ok "${TEST_NAME}" cp "${ERR_PATH}" err-before-scan
-create_test_globalrc '' '
-[authentication]
-    hashes = sha1
-    scan hash = sha1'
-run_ok "${TEST_NAME}" cylc scan -fb -n "${SUITE_NAME}" 'localhost'
-comm -13 err-before-scan "${ERR_PATH}" >"${TEST_NAME}-diff"
-# Wrong hash usage should not be logged as the hash choice may change.
-cat "${TEST_NAME}-diff" >/dev/tty
-diff "${TEST_NAME}-diff" - /dev/tty
-cmp_ok "${TEST_NAME}-diff" '/dev/null' \
-    | sed -e 's/.*@localhost://')
-
-# Connect using SHA256 hash.
-create_test_globalrc '' '
-[authentication]
-    hashes = sha256,md5'
-
-# Connect using SHA256 hash.
-TEST_NAME="${TEST_NAME_BASE}-new-scan-md5-sha256"
-run_ok "${TEST_NAME}" cylc trigger "${SUITE_NAME}" bar 1
-grep_ok "INFO - \[client-command\] trigger_task" "$(cylc cat-log -l $SUITE_NAME)"
-
-# Shutdown using SHA256.
-TEST_NAME="${TEST_NAME_BASE}-stop-md5-sha256"
-run_ok "${TEST_NAME}" cylc stop --max-polls=10 --interval=1 "${SUITE_NAME}"
-
-# Double check shutdown.
-TEST_NAME="${TEST_NAME_BASE}-stop-md5"
-sleep 2
-run_fail "${TEST_NAME}" cylc stop --max-polls=10 --interval=1 "${SUITE_NAME}"
-
-# Purge.
-purge_suite "${SUITE_NAME}"
-exit
diff --git a/tests/broadcast/09-remote/suite.rc b/tests/broadcast/09-remote/suite.rc
index fa64eb84b73..7578bf53959 100644
--- a/tests/broadcast/09-remote/suite.rc
+++ b/tests/broadcast/09-remote/suite.rc
@@ -13,7 +13,7 @@
             graph="""t1 => t2"""
 [runtime]
     [[t1]]
-        script = cylc broadcast "${CYLC_SUITE_NAME}" -n t2 -s 'script=true'
+        script = cylc broadcast -v -v --debug "${CYLC_SUITE_NAME}" -n t2 -s 'script=true'
         [[[remote]]]
             host = {{environ["CYLC_TEST_HOST"]}}
     [[t2]]
diff --git a/tests/lib/bash/test_header b/tests/lib/bash/test_header
index 89dc1036d22..92cd9710d13 100644
--- a/tests/lib/bash/test_header
+++ b/tests/lib/bash/test_header
@@ -195,7 +195,7 @@ function cmp_ok() {
     local FILE_CONTROL=${2:--}
     local TEST_NAME=$(basename $FILE_TEST)-cmp-ok
     local DIFF_CMD=${CYLC_TEST_DIFF_CMD:-'diff -u'}
-    if ${DIFF_CMD} "$FILE_TEST" "$FILE_CONTROL" >"${TEST_NAME}.stderr" 2>&1;then
+    if ${DIFF_CMD} "$FILE_CONTROL" "$FILE_TEST" >"${TEST_NAME}.stderr" 2>&1;then
         ok $TEST_NAME
         return
     else
diff --git a/tests/restart/broadcast/suite.rc b/tests/restart/broadcast/suite.rc
index ef33ec84ca0..0fef3d6f90f 100644
--- a/tests/restart/broadcast/suite.rc
+++ b/tests/restart/broadcast/suite.rc
@@ -4,7 +4,7 @@
     UTC mode = True
     [[event hooks]]
         timeout handler = shutdown_this_suite_hook
-        timeout = PT3M
+        timeout = PT1M
 [scheduling]
     initial cycle time = 20130923T00
     final cycle time   = 20130923T00