Add a livestream and more...

.github/workflows/CI.yaml

@@ -0,0 +1,33 @@
+name: CI
+
+on:
+  push:
+    paths: '**'
+
+jobs:
+  push_x64:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      # - name: Login to Docker Hub
+      #   uses: docker/login-action@v1
+      #   with:
+      #     username: ${{ secrets.DOCKERHUB_USERNAME }}
+      #     password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - uses: satackey/action-docker-layer-caching@v0.0.11
+        continue-on-error: true
+      - name: Create empty settings.env file
+        run: touch settings.env
+      - name: Build containers
+        run: make build
+      - name: Push containers
+        run: make push
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: satackey/action-docker-layer-caching@v0.0.11
+        continue-on-error: true
+      - name: Run pytest
+        run: make test

.gitignore

@@ -28,3 +28,6 @@ camera-app/tests/test_assets/*
 */client_secrets.json
 */env_params.sh
 settings.env
+
+# Proxy
+proxy/logs/*

Makefile

@@ -11,12 +11,23 @@ help:
 up:
 	docker-compose up
 
+down:
+	docker-compose down
+
 start:
 	docker-compose up --detach
 
 build:
 	docker-compose build --parallel
 
+push:
+	docker-compose push
+
+test:
+	docker-compose run camera pytest
+
 create_db:
 	docker-compose run webapp python3 util.py create_db
 
+train_local:
+	docker-compose run camera python3 notebooks/local_prepare_train.py

camera-app/birbcam.py

@@ -18,6 +18,8 @@
 import time
 import traceback
 from fastai.vision.all import *
+from flask_opencv_streamer.streamer import Streamer
+
 
 
 # Set up logging
@@ -40,6 +42,16 @@
 REGION_NAME = os.getenv("REGION_NAME", "Canada")
 MIN_CORRECT_CONF = os.getenv("MIN_CORRECT_CONF", 0.75)
 MIN_UNREVIEWED_CONF = os.getenv("MIN_UNREVIEWED_CONF", 0.9)
+WORKER_PROC = int(os.getenv("BIRBCAM_WORKER_PROC", "2"))
+
+# Streamer
+ENABLE_STREAMER = os.getenv("ENABLE_STREAMER", False)
+
+streamer_port = 3030
+streamer_require_login = False
+
+if ENABLE_STREAMER:
+    streamer = Streamer(streamer_port, streamer_require_login)
 
 # Database path
 DB_PATH = os.getenv('DB_PATH', '../data/model_results.db')
@@ -55,7 +67,7 @@
 utc_tz = pytz.timezone('UTC')
 
 # Datetime format for printing and string representation
-dt_fmt = '%Y-%m-%dT%H:%M:%S'
+dt_fmt = '%Y-%m-%dT%H:%M:%S.%f'
 
 # Where to save captured frames with changes
 save_dir = os.path.join(DATA_DIR, 'imgs/')
@@ -65,49 +77,71 @@
 
 # Create the Videoapture object for the webcam
 capture = cv.VideoCapture()
-capture.set(cv.CAP_PROP_FPS, 1)
+
+# Camera and video settings
+capture.set(cv.CAP_PROP_FPS, int(os.getenv("BIRBCAM_CAMERA_FPS", "1"))) # Camera FPS
+BIRBCAM_DETECTION_RATE = int(os.getenv("BIRBCAM DETECTION RATE", "1")) # Divisor for how many camera frames go to detector
+CV_MASK_THRESH = 255   # Threhold for foreground mask: 255 indicates objects
+CV_KERNEL_SIZE = 25    # nxn kernel size for 2d median filter on foreground mask
 
 # Create the astral city location object
 city = LocationInfo(LOCATION_NAME, REGION_NAME, tz, BIRBCAM_LATITUDE, BIRBCAM_LONGITUDE) 
 
 
-def camera_loop(queue, stop_time):
+def camera_loop(streamer_queue, stop_time):
     # Model params
     # https://docs.opencv.org/master/d1/dc5/tutorial_background_subtraction.html
-    mask_thresh = 255   # Threhold for foreground mask: 255 indicates objects
-    kernel_size = 25    # nxn kernel size for 2d median filter on foreground mask
     lr = 0.05           # learning rate for the background sub model
     burn_in = 30        # frames of burn in for the background sub model
     i = 0               # burn in iteration tracker
+    frame_number = 0
     # Create the background subtraction model
     backSub = cv.createBackgroundSubtractorMOG2()
     #backSub = cv.createBackgroundSubtractorKNN()
+
+    # Create the workers for the processing pool
+    pool = mp.Pool(WORKER_PROC)
+    manager = mp.Manager()
+    queue = manager.Queue()
+    workers = []
+    for i in range(WORKER_PROC):
+        workers.append(
+            pool.apply_async(image_processor, (queue,))
+        )
+
     # Open the webcam
     capture.open(0)
+
     current_time = dt.datetime.now(tz=tz)
     while current_time <= stop_time:
         current_time = dt.datetime.now(tz=tz)
         timestamp = current_time.strftime(dt_fmt)
-        utc_timestamp = dt.datetime.now(tz=utc_tz).strftime(dt_fmt)
+        utc_timestamp = dt.datetime.now(tz=utc_tz).strftime(dt_fmt)[:-5]
         ret, frame = capture.read()
         if ret:
+            frame_number += 1
             if bool(ROTATE_CAMERA):
                 frame = np.rot90(frame, k=-1)
-            fgMask = backSub.apply(frame, learningRate=lr)
-            if i < burn_in:
-                i += 1
-                continue
+            if ENABLE_STREAMER:
+                streamer_queue.put(frame)
+            if frame_number % BIRBCAM_DETECTION_RATE == 0:
+                fgMask = backSub.apply(frame, learningRate=lr)
+                if i < burn_in:
+                    i += 1
+                    continue
 
-            # Threshold mask
-            fgMaskMedian = medfilt2d(fgMask, kernel_size)
-            if (fgMaskMedian >= mask_thresh).any():
-                # Put the frame and corresponding timestamp into the 
-                # queue to be processed by the fastai models
-                queue.put((frame, timestamp, utc_timestamp))
-                logging.info(f'Passed image with timestamp {timestamp} for processing')
+                # Threshold mask
+                fgMaskMedian = medfilt2d(fgMask, CV_KERNEL_SIZE)
+                if (fgMaskMedian >= CV_MASK_THRESH).any():
+                    # Put the frame and corresponding timestamp into the
+                    # queue to be processed by the fastai models
+                    queue.put((frame, timestamp, utc_timestamp))
+                    logging.info(f'Passed image with timestamp {timestamp} for processing')
 
     # Release the webcam        
     capture.release()
+    # Finish up workers
+    pool.close()
 
 
 def prediction_cleanup():
@@ -150,13 +184,28 @@ def night_pause_loop(stop_time):
     prediction_cleanup()
     current_time = dt.datetime.now(tz=tz)
     while current_time < stop_time:
+        # logarithmic sleeping to reduce iterations
         current_time = dt.datetime.now(tz=tz)
         time_diff_seconds = (stop_time - current_time).total_seconds()
-        # logarithmic sleeping to reduce iterations
         time.sleep((time_diff_seconds // 2) + 1)
 
+def streamer_loop(streamer_queue):
+    while True:
+        try:
+            try:
+                frame = streamer_queue.get()
+            except BrokenPipeError:
+                logging.info('Streamer queue borked...')
+                time.sleep(5)
+                continue
+            streamer.update_frame(frame)
+            if not streamer.is_streaming:
+                streamer.start_streaming()
+        except Exception as e:
+            logging.error(traceback.format_exc())
+            pass
 
-def main_loop(queue):
+def main_loop(streamer_queue):
     while True:
         try:
             # Figure out when to run the webcam based on dawn and dusk today
@@ -173,7 +222,7 @@ def main_loop(queue):
             # We can capture images, start the camera loop until sunset today
             elif current_time >= today_start and current_time <= today_end:
                 logging.info(f'Capturing images until dusk at {today_end:{dt_fmt}}')
-                camera_loop(queue, today_end)
+                camera_loop(streamer_queue, today_end)
 
             # Pause image capture until dawn tomorrow
             elif current_time > today_end:
@@ -192,7 +241,11 @@ def image_processor(queue, DB_PATH=DB_PATH, save_dir=save_dir, model_path=MODEL_
     x = None
     while True:
         try:
-            x = queue.get()
+            try:
+                x = queue.get()
+            except BrokenPipeError:
+                # Queue is finished so exit
+                return
             # Get the frame and timestamp for the image to be processed
             frame, timestamp, utc_timestamp = x
             logging.debug(f'Processing image with timestamp {timestamp}')
@@ -217,7 +270,7 @@ def image_processor(queue, DB_PATH=DB_PATH, save_dir=save_dir, model_path=MODEL_
             # Write results to sqlite3 database
             conn = sqlite3.connect(DB_PATH, timeout=60)
             with conn:
-                conn.execute("INSERT INTO results VALUES (?,?,?,?,?,?,?)", 
+                conn.execute("INSERT INTO results VALUES (?,?,?,?,?,?,?)",
                     (utc_timestamp, timestamp, filename, pred_label, confidence, None, None))
             conn.close()
             logging.info(f'Processed image with timestamp {timestamp} and found label(s) {pred_label}')
@@ -230,14 +283,17 @@ def main():
     # Use a multiprocessing queue to offload slow image processing 
     # to other processes/cores and keep the camera_loop from being 
     # blocked and missing frames.
-    q = mp.Queue()
-    p1 = mp.Process(target=main_loop, args=(q,))
-    p2 = mp.Process(target=image_processor, args=(q,))
+    m = mp.Manager()
+    streamer_queue = m.Queue()
+    p1 = mp.Process(target=main_loop, args=(streamer_queue,))
+    if ENABLE_STREAMER:
+        streamer_process = mp.Process(target=streamer_loop, args=(streamer_queue,))
     p1.start()
-    p2.start()
+    if ENABLE_STREAMER:
+        streamer_process.start()
     p1.join()
-    p2.join()
-
+    if ENABLE_STREAMER:
+        streamer_process.join()
 
 if __name__ == '__main__':
     main()

camera-app/requirements.txt

@@ -2,6 +2,7 @@ astral==2.2
 chardet==3.0.4
 click==7.1.2
 colorama==0.4.4
+flask-opencv-streamer==1.4 
 google-api-core==1.26.0
 google-api-python-client==1.12.8
 googleapis-common-protos==1.52.0

camera-app/tests/test_birbcam.py

@@ -6,8 +6,9 @@
 import sqlite3
 import time
 
-from birbcam import image_processor
-from birbcam import dt_fmt, tz, utc_tz
+
+from ..birbcam import image_processor
+from ..birbcam import dt_fmt, tz, utc_tz
 
 
 def test_image_processor():
@@ -43,4 +44,4 @@ def test_image_processor():
 
     assert count == 1, "Did not find the expected row in the database!"
 
-    return
+    return

docker-compose.yml

@@ -10,9 +10,13 @@ services:
       - ./camera-app/:/usr/app/
       - /opt/vc:/opt/vc
       - ./data/:/data/
+      - ./notebooks/:/usr/app/notebooks/
+    ports:
+      - "9090:3030"
     devices:
       - /dev/video0:/dev/video0
-    command: python3 /usr/app/birbcam.py
+    command: python3 birbcam.py
+    working_dir: /usr/app/
     environment:
       - DB_PATH=/data/model.db
       - MODEL_PATH=/data/birbcam_prod.pkl
@@ -33,8 +37,6 @@ services:
     volumes:
       - ./webapp/birbcam-app:/app
       - ./data/:/data/
-    ports:
-      - "8080:5000"
     env_file:
       - settings.env
     environment:
@@ -45,3 +47,12 @@ services:
       - DATA_DIR=/data
     # runtime: "nvidia"
     restart: always
+
+  proxy:
+    image: nginx:stable
+    init: true
+    ports:
+      - "8080:8080"
+    volumes:
+      - ./proxy/nginx.conf:/etc/nginx/nginx.conf:ro
+      - ./proxy/logs/:/var/log/nginx/