Add graph lock

helm/orca/config/alerts-mapping.yaml

@@ -61,6 +61,11 @@ prometheus:
         origin: kubernetes
         kind: cluster
         properties: {}
+    - name: KubeAPIErrorsHigh
+      source_mapping:
+        origin: kubernetes
+        kind: cluster
+        properties: {}
     - name: KubeHpaMaxedOut
       source_mapping:
         origin: kubernetes

orca/api/manager.py

@@ -0,0 +1,51 @@
+# Copyright 2020 OpenRCA Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import multiprocessing
+
+import cotyledon
+import flask
+
+from orca import api
+from orca.graph import graph
+
+
+class Manager(cotyledon.ServiceManager):
+
+    def initialize(self):
+        graph_lock = multiprocessing.Lock()
+        self.add(APIService, workers=1, args=(graph_lock,))
+
+
+class APIService(cotyledon.Service):
+
+    def __init__(self, worker_id, graph_lock):
+        super().__init__(worker_id)
+        self._graph_lock = graph_lock
+        self.__graph = None
+
+    @property
+    def _graph(self):
+        if not self.__graph:
+            self.__graph = graph.Graph.get(self._graph_lock)
+        return self.__graph
+
+    def run(self):
+        app = self._initialize_application()
+        app.run(host='0.0.0.0')
+
+    def _initialize_application(self):
+        app = flask.Flask(__name__)
+        app.register_blueprint(api.initialize(self._graph))
+        return app

orca/cmd/api.py

@@ -12,9 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from orca.api.app import create_app
+from orca.api import manager
 
 
 def main():
-    app = create_app()
-    app.run(host='0.0.0.0')
+    api_manager = manager.Manager()
+    api_manager.initialize()
+    api_manager.run()

orca/graph/graph.py

@@ -15,11 +15,14 @@
 import abc
 import enum
 import uuid
+import contextlib
 
 import addict as dictlib
 
-from orca.common import logger
+from orca.common import config, logger
+from orca.graph import drivers
 
+CONFIG = config.CONFIG
 LOG = logger.get_logger(__name__)
 
 
@@ -57,10 +60,25 @@ def __repr__(self):
 
 class Graph(object):
 
-    def __init__(self, driver):
+    def __init__(self, driver, lock):
         self._driver = driver
+        self._lock = lock
         self._listeners = []
 
+    @contextlib.contextmanager
+    def locked(self):
+        self.lock()
+        try:
+            yield self
+        finally:
+            self.unlock()
+
+    def lock(self):
+        self._lock.acquire()
+
+    def unlock(self):
+        self._lock.release()
+
     def get_nodes(self, origin=None, kind=None, properties=None):
         return self._driver.get_nodes(origin, kind, properties)
 
@@ -133,6 +151,11 @@ def _notify_listeners(self, event_type, graph_obj):
             else:
                 raise Exception("Unknown event type: %s" % event_type)
 
+    @classmethod
+    def get(cls, lock):
+        driver = drivers.DriverFactory.get(CONFIG.graph.driver)
+        return cls(driver, lock)
+
     @staticmethod
     def create_node(id, properties, origin, kind):
         return Node(id, properties, origin, kind)

orca/topology/alerts/elastalert/linker.py

@@ -0,0 +1,24 @@
+# Copyright 2020 OpenRCA Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from orca.topology.alerts import linker
+
+
+class AlertLinker(linker.AlertLinker):
+
+    """Links Alert entities."""
+
+    @classmethod
+    def get(cls, graph):
+        return super().get(graph, 'elastalert')

orca/topology/alerts/elastalert/manager.py

@@ -12,25 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from orca.topology import ingestor, utils
-from orca.topology.alerts import extractor, linker
+from orca.topology import ingestor
+from orca.topology.alerts import extractor
 from orca.topology.alerts.elastalert import extractor as es_extractor
+from orca.topology.alerts.elastalert import linker
 
 
 def initialize_probes(graph):
     return []
 
 
 def initialize_linkers(graph):
-    return [
-        linker.Linker(
-            graph=graph,
-            source_spec=utils.NodeSpec(origin='elastalert', kind='alert'),
-            target_spec=utils.NodeSpec(origin='any', kind='any'),
-            matcher=linker.AlertToSourceMatcher(),
-            bidirectional=False
-        )
-    ]
+    return [linker.AlertLinker.get(graph)]
 
 
 def initialize_handler(graph):

orca/topology/alerts/extractor.py

@@ -70,6 +70,8 @@ def _build_id(self, name, source_mapping):
 
 class SourceMapper(object):
 
+    """Retrieves source mapping for given alert entities."""
+
     def __init__(self, mapping_key):
         self._mapping_key = mapping_key
         self.__mapping = None

orca/topology/alerts/falco/linker.py

@@ -0,0 +1,24 @@
+# Copyright 2020 OpenRCA Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from orca.topology.alerts import linker
+
+
+class AlertLinker(linker.AlertLinker):
+
+    """Links Alert entities."""
+
+    @classmethod
+    def get(cls, graph):
+        return super().get(graph, 'falco')

orca/topology/alerts/falco/manager.py

@@ -12,25 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from orca.topology import ingestor, utils
-from orca.topology.alerts import extractor, linker
+from orca.topology import ingestor
+from orca.topology.alerts import extractor
 from orca.topology.alerts.falco import extractor as falco_extractor
+from orca.topology.alerts.falco import linker
 
 
 def initialize_probes(graph):
     return []
 
 
 def initialize_linkers(graph):
-    return [
-        linker.Linker(
-            graph=graph,
-            source_spec=utils.NodeSpec(origin='falco', kind='alert'),
-            target_spec=utils.NodeSpec(origin='any', kind='any'),
-            matcher=linker.AlertToSourceMatcher(),
-            bidirectional=False
-        )
-    ]
+    return [linker.AlertLinker.get(graph)]
 
 
 def initialize_handler(graph):

orca/topology/alerts/linker.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-from orca.topology import linker
+from orca.topology import linker, utils
+from orca.topology.alerts import matcher
 
 
 class Linker(linker.Linker):
@@ -31,21 +31,15 @@ def _get_linked_nodes(self, alert_node):
             properties=source_mapping.properties)
 
 
-class Matcher(linker.Matcher):
-
-    """Base class for alert matchers."""
-
-
-class AlertToSourceMatcher(Matcher):
+class AlertLinker(Linker):
 
-    """Generic matcher for links between Alert and source objects."""
+    """Links Alert entities."""
 
-    def are_linked(self, alert, obj):
-        source_mapping = alert.properties.source_mapping
-        if not source_mapping.origin == obj.origin:
-            return False
-        if not source_mapping.kind == obj.kind:
-            return False
-        mapping_items = source_mapping.properties.items()
-        obj_items = obj.properties.items()
-        return all(item in obj_items for item in mapping_items)
+    @classmethod
+    def get(cls, graph, origin):
+        return cls(
+            graph=graph,
+            source_spec=utils.NodeSpec(origin=origin, kind='alert'),
+            target_spec=utils.NodeSpec(origin='any', kind='any'),
+            matcher=matcher.AlertToSourceMatcher(),
+            bidirectional=False)

orca/topology/alerts/matcher.py

@@ -0,0 +1,35 @@
+# Copyright 2020 OpenRCA Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from orca.topology import matcher
+
+
+class Matcher(matcher.Matcher):
+
+    """Base class for Alert matchers."""
+
+
+class AlertToSourceMatcher(Matcher):
+
+    """Generic matcher for links between Alert and source objects."""
+
+    def are_linked(self, alert, obj):
+        source_mapping = alert.properties.source_mapping
+        if not source_mapping.origin == obj.origin:
+            return False
+        if not source_mapping.kind == obj.kind:
+            return False
+        mapping_items = source_mapping.properties.items()
+        obj_items = obj.properties.items()
+        return all(item in obj_items for item in mapping_items)

orca/topology/alerts/prometheus/linker.py

@@ -0,0 +1,24 @@
+# Copyright 2020 OpenRCA Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from orca.topology.alerts import linker
+
+
+class AlertLinker(linker.AlertLinker):
+
+    """Links Alert entities."""
+
+    @classmethod
+    def get(cls, graph):
+        return super().get(graph, 'prometheus')