Merge pull request #16 from thread/feeds

Feeds

.circleci/config.yml

@@ -45,15 +45,15 @@ jobs:
       # Run tox, caching the .tox directory
       - restore_cache:
           name: Restore .tox cache
-          key: deps-tox-{{ .Branch }}-{{ checksum "scripts/linting/requirements.txt" }}-{{ checksum "scripts/testing/requirements.txt" }}-{{ checksum "setup.py" }}
+          key: deps-tox-{{ .Branch }}-{{ checksum "scripts/linting/requirements.txt" }}-{{ checksum "scripts/typechecking/requirements.txt" }}-{{ checksum "scripts/testing/requirements.txt" }}-{{ checksum "setup.py" }}
       - run:
           name: Test/Lint/Typecheck
           command: |
             . venv/bin/activate
             tox
       - save_cache:
           name: Save .tox cache
-          key: deps-tox-{{ .Branch }}-{{ checksum "scripts/linting/requirements.txt" }}-{{ checksum "scripts/testing/requirements.txt" }}-{{ checksum "setup.py" }}
+          key: deps-tox-{{ .Branch }}-{{ checksum "scripts/linting/requirements.txt" }}-{{ checksum "scripts/typechecking/requirements.txt" }}-{{ checksum "scripts/testing/requirements.txt" }}-{{ checksum "setup.py" }}
           paths:
             - ".tox"
 

routemaster/config/__init__.py

@@ -1,6 +1,7 @@
 """Loading of application configuration."""
 
 from routemaster.config.model import (
+    Feed,
     Gate,
     State,
     Action,
@@ -24,6 +25,7 @@
 __all__ = (
     'load_config',
     'load_database_config',
+    'Feed',
     'Gate',
     'State',
     'Action',

routemaster/config/loader.py

@@ -11,6 +11,7 @@
 import jsonschema.exceptions
 
 from routemaster.config.model import (
+    Feed,
     Gate,
     State,
     Action,
@@ -106,15 +107,27 @@ def _load_state_machine(
     name: str,
     yaml_state_machine: Yaml,
 ) -> StateMachine:
+    feeds = [_load_feed(x) for x in yaml_state_machine.get('feeds', [])]
+
+    if len(set(x.name for x in feeds)) < len(feeds):
+        raise ConfigError(
+            f"Feeds must have unique names at {'.'.join(path + ['feeds'])}",
+        )
+
     return StateMachine(
         name=name,
         states=[
             _load_state(path + ['states', str(idx)], yaml_state)
             for idx, yaml_state in enumerate(yaml_state_machine['states'])
         ],
+        feeds=feeds,
     )
 
 
+def _load_feed(yaml: Yaml) -> Feed:
+    return Feed(name=yaml['name'], url=yaml['url'])
+
+
 def _load_state(path: Path, yaml_state: Yaml) -> State:
     if 'action' in yaml_state and 'gate' in yaml_state:  # pragma: no branch
         raise ConfigError(  # pragma: no cover

routemaster/config/model.py

@@ -13,7 +13,10 @@
     NamedTuple,
 )
 
-from routemaster.exit_conditions import Context, ExitConditionProgram
+from routemaster.exit_conditions import ExitConditionProgram
+
+if False:  # typing
+    from routemaster.context import Context  # noqa
 
 
 class TimeTrigger(NamedTuple):
@@ -69,7 +72,7 @@ class ContextNextStates(NamedTuple):
     path: str
     destinations: Iterable[ContextNextStatesOption]
 
-    def next_state_for_label(self, label_context: Context) -> str:
+    def next_state_for_label(self, label_context: 'Context') -> str:
         """Returns next state based on context value at `self.path`."""
         val = label_context.lookup(self.path.split('.'))
         for destination in self.destinations:
@@ -133,10 +136,19 @@ class Action(NamedTuple):
 State = Union[Action, Gate]
 
 
+class Feed(NamedTuple):
+    """
+    The definition of a feed of dynamic data to be included in a context.
+    """
+    name: str
+    url: str
+
+
 class StateMachine(NamedTuple):
     """A state machine."""
     name: str
     states: List[State]
+    feeds: List[Feed]
 
     def get_state(self, state_name: str) -> State:
         """Get the state object for a given state name."""

routemaster/config/schema.yaml

@@ -20,6 +20,21 @@ properties:
       title: State Machine
       type: object
       properties:
+        feeds:
+          title: Feeds
+          type: array
+          items:
+            title: Feed
+            type: object
+            properties:
+              name:
+                type: string
+              url:
+                type: string
+            required:
+              - name
+              - url
+            additionalProperties: false
         states:
           title: States
           type: array

routemaster/config/tests/test_loading.py

@@ -8,6 +8,7 @@
 import pytest
 
 from routemaster.config import (
+    Feed,
     Gate,
     Action,
     Config,
@@ -45,6 +46,7 @@ def test_trivial_config():
         state_machines={
             'example': StateMachine(
                 name='example',
+                feeds=[],
                 states=[
                     Gate(
                         name='start',
@@ -73,6 +75,9 @@ def test_realistic_config():
         state_machines={
             'example': StateMachine(
                 name='example',
+                feeds=[
+                    Feed(name='data_feed', url='http://localhost/<label>'),
+                ],
                 states=[
                     Gate(
                         name='start',
@@ -190,6 +195,7 @@ def test_next_states_shorthand_results_in_constant_config():
         state_machines={
             'example': StateMachine(
                 name='example',
+                feeds=[],
                 states=[
                     Gate(
                         name='start',
@@ -224,6 +230,9 @@ def test_environment_variables_override_config_file_for_database_config():
         state_machines={
             'example': StateMachine(
                 name='example',
+                feeds=[
+                    Feed(name='data_feed', url='http://localhost/<label>'),
+                ],
                 states=[
                     Gate(
                         name='start',
@@ -300,5 +309,10 @@ def test_environment_variables_override_config_file_for_database_config():
 
 def test_raises_for_unparseable_database_port_in_environment_variable():
     with mock.patch.dict(os.environ, {'DB_PORT': 'not an int'}):
-        with assert_config_error(f"Could not parse DB_PORT as an integer: 'not an int'."):
+        with assert_config_error("Could not parse DB_PORT as an integer: 'not an int'."):
             load_config(yaml_data('realistic'))
+
+
+def test_multiple_feeds_same_name_invalid():
+    with assert_config_error("Feeds must have unique names at state_machines.example.feeds"):
+        load_config(yaml_data('multiple_feeds_same_name_invalid'))

routemaster/config/tests/test_next_states.py

@@ -9,7 +9,7 @@
     ContextNextStates,
     ContextNextStatesOption,
 )
-from routemaster.exit_conditions import Context
+from routemaster.context import Context
 
 UTC_NOW = datetime.datetime.now(dateutil.tz.tzutc())
 
@@ -36,7 +36,7 @@ def test_context_next_states():
         ],
     )
 
-    context = Context({'foo': True}, UTC_NOW, None)
+    context = Context('label1', {'foo': True}, UTC_NOW, None, [])
 
     assert next_states.all_destinations() == ['1', '2']
     assert next_states.next_state_for_label(context) == '1'
@@ -51,7 +51,7 @@ def test_context_next_states_raises_for_no_valid_state():
         ],
     )
 
-    context = Context({'foo': 'bar'}, UTC_NOW, None)
+    context = Context('label1', {'foo': 'bar'}, UTC_NOW, None, [])
 
     with pytest.raises(RuntimeError):
         next_states.next_state_for_label(context)

routemaster/conftest.py

@@ -33,6 +33,7 @@
 TEST_STATE_MACHINES = {
     'test_machine': StateMachine(
         name='test_machine',
+        feeds=[],
         states=[
             Gate(
                 name='start',

routemaster/context.py

@@ -0,0 +1,76 @@
+"""Context definition for exit condition programs."""
+import datetime
+from typing import Any, Dict, Iterable, Sequence
+
+from routemaster.feeds import Feed
+from routemaster.utils import get_path
+
+
+class Context(object):
+    """Execution context for exit condition programs."""
+
+    def __init__(
+        self,
+        label: str,
+        metadata: Dict[str, Any],
+        now: datetime.datetime,
+        feeds: Dict[str, Feed],
+        accessed_variables: Iterable[str],
+    ) -> None:
+        """Create an execution context."""
+        if now.tzinfo is None:
+            raise ValueError(
+                "Cannot evaluate exit conditions with naive datetimes",
+            )
+
+        self.now = now
+        self.metadata = metadata
+        self.feeds = feeds
+
+        self._pre_warm_feeds(label, accessed_variables)
+
+    def lookup(self, path: Sequence[str]) -> Any:
+        """Look up a path in the execution context."""
+        location, *rest = path
+
+        try:
+            return {
+                'metadata': self._lookup_metadata,
+                'feeds': self._lookup_feed_data,
+            }[location](rest)
+        except (KeyError, ValueError):
+            return None
+
+    def _lookup_metadata(self, path: Sequence[str]) -> Any:
+        return get_path(path, self.metadata)
+
+    def _lookup_feed_data(self, path: Sequence[str]) -> Any:
+        feed_name, *rest = path
+        return self.feeds[feed_name].lookup(rest)
+
+    def property_handler(self, property_name, value, **kwargs):
+        """Handle a property in execution."""
+        if property_name == ('passed',):
+            epoch = kwargs['since']
+            return (self.now - epoch).total_seconds() >= value
+        if property_name == ('defined',):
+            return value is not None
+        if property_name == () and 'in' in kwargs:
+            return value in kwargs['in']
+        raise ValueError("Unknown property {name}".format(
+            name='.'.join(property_name)),
+        )
+
+    def _pre_warm_feeds(self, label: str, accessed_variables: Iterable[str]):
+        for accessed_variable in accessed_variables:
+            parts = accessed_variable.split('.')
+
+            if len(parts) < 2:
+                continue
+
+            if parts[0] != 'feeds':
+                continue
+
+            feed = self.feeds.get(parts[1])
+            if feed is not None:
+                feed.prefetch(label)

routemaster/exit_conditions/__init__.py

@@ -1,9 +1,7 @@
 """Parsing and evaluation of exit condition programs."""
 
-from routemaster.exit_conditions.context import Context
 from routemaster.exit_conditions.program import ExitConditionProgram
 
 __all__ = (
-    'Context',
     'ExitConditionProgram',
 )

routemaster/exit_conditions/program.py

@@ -3,7 +3,6 @@
 import typing
 
 from routemaster.exit_conditions.parser import parse
-from routemaster.exit_conditions.context import Context
 from routemaster.exit_conditions.analysis import find_accessed_keys
 from routemaster.exit_conditions.peephole import peephole_optimise
 from routemaster.exit_conditions.evaluator import evaluate
@@ -12,6 +11,9 @@
     format_parse_error_message,
 )
 
+if False:  # typing
+    from routemaster.context import Context  # noqa
+
 
 class ExitConditionProgram(object):
     """Compiled exit condition program."""
@@ -39,7 +41,7 @@ def accessed_variables(self) -> typing.Iterable[str]:
         for accessed_key in find_accessed_keys(self._instructions):
             yield '.'.join(accessed_key)
 
-    def run(self, context: Context) -> bool:
+    def run(self, context: 'Context') -> bool:
         """Evaluate this program with a given context."""
         return evaluate(
             self._instructions,