Merge branch 'master' into task-logging-test

.coveragerc

@@ -0,0 +1,28 @@
+[run]
+branch = True
+source = ./labthings
+omit = .venv/*, labthings/server/wsgi/*, , labthings/server/monkey.py
+concurrency = greenlet
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:
+
+ignore_errors = True
+
+[html]
+directory = coverage_html_report

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 88
+exclude = tests/*

.github/workflows/ci.yml

@@ -0,0 +1,52 @@
+name: CI
+
+on: [push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+
+      - name: Install Poetry
+        uses: dschep/install-poetry-action@v1.3
+
+      - name: Cache Poetry virtualenv
+        uses: actions/cache@v1
+        id: cache
+        with:
+          path: ~/.virtualenvs
+          key: poetry-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            poetry-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Set Poetry config
+        run: |
+          poetry config virtualenvs.in-project false
+          poetry config virtualenvs.path ~/.virtualenvs
+
+      - name: Install Dependencies
+        run: poetry install
+        if: steps.cache.outputs.cache-hit != 'true'
+
+      - name: Code Quality
+        run: poetry run black . --check
+
+      - name: Test with pytest
+        run: poetry run pytest --cov-report term-missing --cov-report=xml --cov=labthings ./tests
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./coverage.xml
+          flags: unittests
+          name: codecov-umbrella
+          fail_ci_if_error: true

.gitignore

@@ -39,6 +39,7 @@ pip-delete-this-directory.txt
 
 # Unit test / coverage reports
 htmlcov/
+coverage_html_report/
 .tox/
 .nox/
 .coverage
@@ -50,6 +51,8 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+coverage_html_report/
+prof/
 
 # Translations
 *.mo

README.md

@@ -3,7 +3,8 @@
 [![LabThings](https://img.shields.io/badge/-LabThings-8E00FF?style=flat&logo=data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4NCjwhRE9DVFlQRSBzdmcgIFBVQkxJQyAnLS8vVzNDLy9EVEQgU1ZHIDEuMS8vRU4nICAnaHR0cDovL3d3dy53My5vcmcvR3JhcGhpY3MvU1ZHLzEuMS9EVEQvc3ZnMTEuZHRkJz4NCjxzdmcgY2xpcC1ydWxlPSJldmVub2RkIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiIHN0cm9rZS1taXRlcmxpbWl0PSIyIiB2ZXJzaW9uPSIxLjEiIHZpZXdCb3g9IjAgMCAxNjMgMTYzIiB4bWw6c3BhY2U9InByZXNlcnZlIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjxwYXRoIGQ9Im0xMjIuMjQgMTYyLjk5aDQwLjc0OHYtMTYyLjk5aC0xMDEuODd2NDAuNzQ4aDYxLjEyMnYxMjIuMjR6IiBmaWxsPSIjZmZmIi8+PHBhdGggZD0ibTAgMTIuMjI0di0xMi4yMjRoNDAuNzQ4djEyMi4yNGg2MS4xMjJ2NDAuNzQ4aC0xMDEuODd2LTEyLjIyNGgyMC4zNzR2LTguMTVoLTIwLjM3NHYtOC4xNDloOC4wMTl2LTguMTVoLTguMDE5di04LjE1aDIwLjM3NHYtOC4xNDloLTIwLjM3NHYtOC4xNWg4LjAxOXYtOC4xNWgtOC4wMTl2LTguMTQ5aDIwLjM3NHYtOC4xNWgtMjAuMzc0di04LjE0OWg4LjAxOXYtOC4xNWgtOC4wMTl2LTguMTVoMjAuMzc0di04LjE0OWgtMjAuMzc0di04LjE1aDguMDE5di04LjE0OWgtOC4wMTl2LTguMTVoMjAuMzc0di04LjE1aC0yMC4zNzR6IiBmaWxsPSIjZmZmIi8+PC9zdmc+DQo=)](https://github.com/labthings/)
 [![PyPI](https://img.shields.io/pypi/v/labthings)](https://pypi.org/project/labthings/)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
-[![Gitter](https://badges.gitter.im/labthings/community.svg)](https://gitter.im/labthings/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
+[![codecov](https://codecov.io/gh/labthings/python-labthings/branch/master/graph/badge.svg)](https://codecov.io/gh/labthings/python-labthings)
+[![Riot.im](https://img.shields.io/badge/chat-on%20riot.im-368BD6)](https://riot.im/app/#/room/#labthings:matrix.org)
 
 A Python implementation of the LabThings API structure, based on the Flask microframework.
 

examples/builder.py

@@ -1,19 +1,17 @@
-import uuid
-import types
-import functools
-import atexit
+# Monkey patch for easy concurrency
+from labthings.server.monkey import patch_all
+
+patch_all()
+
+# Import requirements
 import logging
 
 from labthings.server.quick import create_app
-from labthings.server.view.builder import property_of
+from labthings.server.view.builder import property_of, action_from
 
 from components.pdf_component import PdfComponent
 
 
-def cleanup():
-    logging.info("Exiting. Running any cleanup code here...")
-
-
 # Create LabThings Flask app
 app, labthing = create_app(
     __name__,
@@ -42,8 +40,17 @@ def cleanup():
     ),
     "/dictionary",
 )
+labthing.add_view(
+    action_from(
+        my_component.average_data,
+        description="Take an averaged measurement",
+        task=True,  # Is the action a long-running task?
+        safe=True,  # Is the state of the Thing unchanged by calling the action?
+        idempotent=True,  # Can the action be called repeatedly with the same result?
+    ),
+    "/average",
+)
 
-atexit.register(cleanup)
 
 # Start the app
 if __name__ == "__main__":

examples/components/pdf_component.py

@@ -2,6 +2,8 @@
 import math
 import time
 
+from typing import List
+
 """
 Class for our lab component functionality. This could include serial communication,
 equipment API calls, network requests, or a "virtual" device as seen here.
@@ -38,7 +40,7 @@ def data(self):
         """Return a 1D data trace."""
         return [self.noisy_pdf(x) for x in self.x_range]
 
-    def average_data(self, n: int):
+    def average_data(self, n: int = 10, optlist: List[int] = [1, 2, 3]):
         """Average n-sets of data. Emulates a measurement that may take a while."""
         summed_data = self.data
 

examples/simple_extensions.py

@@ -40,7 +40,6 @@ def ext_on_my_component(component):
 
 
 static_folder = path_relative_to(__file__, "static")
-print(static_folder)
 
 example_extension = BaseExtension(
     "org.labthings.examples.extension", static_folder=static_folder

examples/simple_thing.py

@@ -1,6 +1,16 @@
+#!/usr/bin/env python
+from gevent import monkey
+
+# Patch most system modules. Leave threads untouched so we can still use them normally if needed.
+print("Monkey patching with Gevenet")
+monkey.patch_all(thread=False)
+print("Monkey patching successful")
+
 import random
 import math
 import time
+import logging
+import atexit
 
 from labthings.server.quick import create_app
 from labthings.server.decorators import (
@@ -13,7 +23,7 @@
 from labthings.server.view import View
 from labthings.server.find import find_component
 from labthings.server import fields
-from labthings.core.tasks import taskify
+from labthings.core.tasks import taskify, update_task_data
 
 
 """
@@ -22,6 +32,14 @@
 """
 
 
+from gevent.monkey import get_original
+
+get_ident = get_original("_thread", "get_ident")
+
+print(f"ROOT IDENT")
+print(get_ident())
+
+
 class MyComponent:
     def __init__(self):
         self.x_range = range(-100, 100)
@@ -48,8 +66,10 @@ def average_data(self, n: int):
         """Average n-sets of data. Emulates a measurement that may take a while."""
         summed_data = self.data
 
+        logging.warning("Starting an averaged measurement. This may take a while...")
         for i in range(n):
             summed_data = [summed_data[i] + el for i, el in enumerate(self.data)]
+            update_task_data({"data": summed_data})
             time.sleep(0.25)
 
         summed_data = [i / n for i in summed_data]
@@ -150,6 +170,13 @@ def post(self, args):
         return task
 
 
+# Handle exit cleanup
+def cleanup():
+    logging.info("Exiting. Running any cleanup code here...")
+
+
+atexit.register(cleanup)
+
 # Create LabThings Flask app
 app, labthing = create_app(
     __name__,
@@ -173,4 +200,4 @@ def post(self, args):
     from labthings.server.wsgi import Server
 
     server = Server(app)
-    server.run(host="0.0.0.0", port=5000, debug=False)
+    server.run(host="0.0.0.0", port=5000, debug=False, zeroconf=False)

labthings/core/event.py

@@ -0,0 +1,69 @@
+from gevent.hub import getcurrent
+import gevent
+import time
+import logging
+
+from gevent.monkey import get_original
+
+# Guarantee that Task threads will always be proper system threads, regardless of Gevent patches
+Event = get_original("threading", "Event")
+
+
+class ClientEvent(object):
+    """
+    An event-signaller object with per-client setting and waiting.
+
+    A client can be any Greenlet or native Thread. This can be used, for example,
+    to signal to clients that new data is available
+    """
+
+    def __init__(self):
+        self.events = {}
+
+    def wait(self, timeout: int = 5):
+        """Wait for the next data frame (invoked from each client's thread)."""
+        ident = id(getcurrent())
+        if ident not in self.events:
+            # this is a new client
+            # add an entry for it in the self.events dict
+            # each entry has two elements, a threading.Event() and a timestamp
+            self.events[ident] = [Event(), time.time()]
+
+        # We have to reimplement event waiting here as we need native thread events to allow gevent context switching
+        wait_start = time.time()
+        while not self.events[ident][0].is_set():
+            now = time.time()
+            if now - wait_start > timeout:
+                return False
+            gevent.sleep(0)
+        return True
+
+    def set(self, timeout=5):
+        """Signal that a new frame is available."""
+        now = time.time()
+        remove = None
+        for ident, event in self.events.items():
+            if not event[0].is_set():
+                # if this client's event is not set, then set it
+                # also update the last set timestamp to now
+                event[0].set()
+                event[1] = now
+            else:
+                # if the client's event is already set, it means the client
+                # did not process a previous frame
+                # if the event stays set for more than `timeout` seconds, then assume
+                # the client is gone and remove it
+                if now - event[1] >= timeout:
+                    remove = ident
+        if remove:
+            del self.events[remove]
+
+    def clear(self):
+        """Clear frame event, once processed."""
+        ident = id(getcurrent())
+        if ident not in self.events:
+            logging.error(f"Mismatched ident. Current: {ident}, available:")
+            logging.error(self.events.keys())
+            return False
+        self.events[id(getcurrent())][0].clear()
+        return True