Merge pull request #156 from amosproj/develop

Final release merge into main

Dockerfile

@@ -24,8 +24,7 @@ ADD embark/requirements.txt /app/embark/requirements.txt
 
 RUN yes | sudo /app/emba/installer.sh -D -F  && \
     sudo pip3 install uwsgi -I --no-cache-dir && \
-    pip3 install --user --no-warn-script-location -r /app/embark/requirements.txt && \
-    mkdir /app/embark/logs
+    pip3 install --user --no-warn-script-location -r /app/embark/requirements.txt
 
 # 8000 for http workers. 8001 for ws workers
 EXPOSE 8000

docker.md

@@ -14,9 +14,12 @@ Refer to `template.env`, you will see the following env variables
 1. DATABASE_NAME: Name of the sql database
 2. DATABASE_USER: User of the database
 3. DATABASE_PASSWORD: For for logging in to the database
-4. DATABASE_HOST: host of the database
-5. DATABASE_PORT: port
-6. MYSQL_ROOT_PASSWORD: Root password. Always equal to DATABASE_PASSWORD for our dev setups
+4. DATABASE_HOST: Host for MySQL database
+5. DATABASE_PORT: Port for MySQL database
+6. MYSQL_ROOT_PASSWORD: Root password to connect to mysql server
+7. MYSQL_DATABASE: Database that gets created on container startup. Same as DATABASE_NAME
+8. REDIS_HOST: Host for Redis DB
+9. REDIS_PORT: Port for Redis DB
 ``` 
 
 We are not maintaining a central copy for now. Till then please main your own copy wherever you setup your dev environment.
@@ -35,30 +38,19 @@ We are not maintaining a central copy for now. Till then please main your own co
 DATABASE_NAME=<Name you are going to give your db>
 DATABASE_USER=root
 DATABASE_PASSWORD=<value of MYSQL_ROOT_PASSWORD>
-DATABASE_HOST=0.0.0.0(or host.docker.internal for windows)
+DATABASE_HOST=0.0.0.0(or host.docker.internal for windows or MacOs)
 DATABASE_PORT=3306
+REDIS_HOST=0.0.0.0(or host.docker.internal for windows or MacOs)
+REDIS_PORT=6379
 MYSQL_ROOT_PASSWORD=<This should be set>
+MYSQL_DATABASE=<Same as DATABASE_NAME>
 ```
 
 3. Bring your containers up  
 `docker-compose up -d`
-
 
-4. Now you can exec into the mysql container and create your database.
-```
-docker exec -it amos-ss2021-emba-service_auth-db_1 bash
-mysql -p
-```
-When prompted for a password enter the value in `MYSQL_ROOT_PASSWORD`  
-`Create database <value in DATABASE_NAME>`
-
-5. Restart your containers. This additional step is neccessary since there is no dependency between the containers. And django would try to connect to DB not there yet and fail.  
-`docker-compose restart emba`
-
-6. Exec into your emba-django container  
-`docker exec -it amos-ss2021-emba-service_emba_1 bash`
-
-7. Run migrations
+### Run migrations
+To run migrations for any changes in django db models
 ```
 python3 manage.py makemigrations uploader users
 python3 manage.py migrate

embark/embark/consumers.py

@@ -54,6 +54,5 @@ def disconnect(self, close_code):
     def send_message(self, event):
         # Receive message and extract data from room group
         message = event['message']
-        logger.debug(message)
         # Send message to WebSocket
         self.send(json.dumps(message, sort_keys=False))

embark/embark/logreader.py

@@ -1,5 +1,6 @@
 import copy
 import difflib
+import pathlib
 import re
 import time
 
@@ -10,7 +11,7 @@
 from channels.generic.websocket import WebsocketConsumer
 
 from django.conf import settings
-from uploader.models import Firmware
+from uploader.models import Firmware, FirmwareFile
 from inotify_simple import INotify, flags
 from asgiref.sync import async_to_sync
 from channels.layers import get_channel_layer
@@ -28,17 +29,27 @@ def __init__(self, firmware_id):
         # global module count and status_msg directory
         self.module_count = 0
         self.firmware_id = firmware_id
+        self.firmware_id_str = str(self.firmware_id)
+        try:
+            self.firmwarefile = Firmware.objects.get(pk=firmware_id).firmware.__str__()
+        except Exception as e:
+            logger.info(e)
 
         # set variables for channels communication
         self.room_group_name = 'updatesgroup'
         self.channel_layer = get_channel_layer()
 
+        # variables for cleanup
+        self.finish = False
+        self.wd = None
+
         # for testing
         self.test_list1 = []
         self.test_list2 = []
 
         # status update dict (appended to processmap)
         self.status_msg = {
+            "firmwarename": self.firmwarefile,
             "percentage": 0.0,
             "module": "",
             "phase": "",
@@ -53,7 +64,11 @@ def __init__(self, firmware_id):
     def update_status(self, stream_item_list):
         # progress percentage TODO: improve percentage calculation
         self.module_count += 1
-        percentage = self.module_count / 35
+
+        # calculate percentage
+        percentage = self.module_count / 34
+
+        # set attributes of current message
         self.status_msg["module"] = stream_item_list[0]
         self.status_msg["percentage"] = percentage
 
@@ -63,37 +78,51 @@ def update_status(self, stream_item_list):
         # append it to the data structure
         global process_map
         if self.firmware_id > 0:
-            process_map[self.firmware_id].append(tmp_mes)
-
-        # send it to room group
-        if self.firmware_id > 0:
-            async_to_sync(self.channel_layer.group_send)(
-                self.room_group_name, {
-                    "type": 'send.message',
-                    "message": process_map
-                }
-            )
+            found = False
+            for mes in process_map[self.firmware_id_str]:
+                if mes["phase"] == tmp_mes["phase"] and mes["module"] == tmp_mes["module"]:
+                    found = True
+
+            if not found:
+                process_map[self.firmware_id_str].append(tmp_mes)
+
+                # send it to room group
+                if self.firmware_id > 0:
+                    async_to_sync(self.channel_layer.group_send)(
+                        self.room_group_name, {
+                            "type": 'send.message',
+                            "message": process_map
+                        }
+                    )
 
     # update dictionary with phase changes
     def update_phase(self, stream_item_list):
         self.status_msg["phase"] = stream_item_list[1]
 
         # get copy of the current status message
         tmp_mes = copy.deepcopy(self.status_msg)
-        # append it to the data structure
 
+        # append it to the data structure
         global process_map
         if self.firmware_id > 0:
-            process_map[self.firmware_id].append(tmp_mes)
-
-        # send it to room group
-        if self.firmware_id > 0:
-            async_to_sync(self.channel_layer.group_send)(
-                self.room_group_name, {
-                    'type': 'send.message',
-                    'message': process_map
-                }
-            )
+            found = False
+            for mes in process_map[self.firmware_id_str]:
+                if mes["phase"] == tmp_mes["phase"] and mes["module"] == tmp_mes["module"]:
+                    found = True
+
+            if not found:
+                process_map[self.firmware_id_str].append(tmp_mes)
+
+                # send it to room group
+                if self.firmware_id > 0:
+                    async_to_sync(self.channel_layer.group_send)(
+                        self.room_group_name, {
+                            'type': 'send.message',
+                            'message': process_map
+                        }
+                    )
+        if "Test ended" in stream_item_list[1]:
+            self.finish = True
 
     def read_loop(self):
 
@@ -106,20 +135,23 @@ def read_loop(self):
 
         logger.info(f"read loop started for {self.firmware_id}")
 
-        while True:
+        while not self.finish:
+
             # get firmware for id which the BoundedExecutor gave the log_reader
             firmware = Firmware.objects.get(pk=self.firmware_id)
 
             # if file does not exist create it otherwise delete its content
-            open(f"{firmware.path_to_logs}emba_new.log", 'w+')
+            pat = f"/app/emba/{settings.LOG_ROOT}/emba_new_{self.firmware_id}.log"
+            if not pathlib.Path(pat).exists():
+                open(pat, 'w+')
 
             # create an entry for the id in the process map
             global process_map
-            if firmware.id not in process_map.keys():
-                process_map[firmware.id] = []
+            if self.firmware_id_str not in process_map.keys():
+                process_map[self.firmware_id_str] = []
 
             # look for new events
-            got_event = self.inotify_events(f"{firmware.path_to_logs}emba.log")
+            got_event = self.inotify_events(f"{firmware.path_to_logs}/emba.log")
 
             for eve in got_event:
                 for flag in flags.from_mask(eve.mask):
@@ -135,6 +167,18 @@ def read_loop(self):
                         # copy diff to tmp file
                         self.copy_file_content(tmp, firmware.path_to_logs)
 
+        self.cleanup()
+        return
+
+    def cleanup(self):
+
+        """
+            Called when logreader should be cleaned up
+        """
+        # inotify = INotify()
+        # inotify.rm_watch(self.wd)
+        logger.info(f"Log reader cleaned up for {self.firmware_id}")
+
     def process_line(self, inp, pat):
 
         """
@@ -157,7 +201,7 @@ def copy_file_content(self, diff, log_path):
                   :return: None
         """
 
-        with open(f"{log_path}emba_new.log", 'a+') as diff_file:
+        with open(f"/app/emba/{settings.LOG_ROOT}/emba_new_{self.firmware_id}.log", 'a+') as diff_file:
             diff_file.write(diff)
 
     def get_diff(self, log_path):
@@ -170,8 +214,8 @@ def get_diff(self, log_path):
         """
 
         # open the two files to get diff from
-        old_file = open(f"{log_path}emba.log")
-        new_file = open(f"{log_path}emba_new.log")
+        old_file = open(f"{log_path}/emba.log")
+        new_file = open(f"/app/emba/{settings.LOG_ROOT}/emba_new_{self.firmware_id}.log")
 
         diff = difflib.ndiff(old_file.readlines(), new_file.readlines())
         return ''.join(x[2:] for x in diff if x.startswith('- '))

embark/embark/settings.py

@@ -194,6 +194,9 @@
 # URL of Login-Page
 LOGIN_URL = ''
 
+# URL of Logout-Page
+LOGOUT_REDIRECT_URL = ''
+
 # Added for FIle storage to get the path to save Firmware images.
 MEDIA_ROOT = os.path.join('uploadedFirmwareImages')  # media directory in the root directory
 MEDIA_URL = '/uploadedFirmwareImages/'

embark/embark/test_logreader.py

@@ -2,7 +2,7 @@
 from . import logreader
 import logging
 
-from uploader.models import Firmware
+from uploader.models import Firmware, FirmwareFile
 
 logger = logging.getLogger('web')
 
@@ -12,8 +12,7 @@ class test_logreader(TestCase):
     def setUp(self):
         self.log_string1 = "[\x1b[0;33m*\x1b[0m] Tue Jun 15 12:12:26 UTC 2021 - P02_firmware_bin_file_check starting\n"
         self.log_string2 = "[\x1b[0;33m*\x1b[0m] Tue Jun 15 12:12:27 UTC 2021 - P02_firmware_bin_file_check finished\n"
-        self.log_string3 = "[\x1b[0;35m!\x1b[0m] \x1b[0;35m Pre-checking phase ended on Tue Jun 15 12:12:58 UTC 2021 " \
-                           "and took about 00:00:33\n"
+        self.log_string3 = "[\x1b[0;35m!\x1b[0m]\x1b[0;35m Test ended on Sat Jul  3 00:07:10 UTC 2021 and took about 00:34:24\x1b[0m\n"
         self.log_string4 = "[*] Wed Apr 21 15:32:46 UTC 2021 - P05_firmware_bin_extractor starting"
         self.log_list = []
         self.log_list.append(self.log_string1)
@@ -22,7 +21,7 @@ def setUp(self):
         self.log_list.append(self.log_string4)
 
         # creat DB entry
-        test_file = open("test.bin", "w")
+        test_file = FirmwareFile.objects.create()
         firmware = Firmware(firmware=test_file)
         firmware.pk = -1
         firmware.save()
@@ -34,5 +33,5 @@ def test_in_pro(self):
         for line in self.log_list:
             res1, res2 = lr.produce_test_output(line)
         self.assertEqual("P02_firmware_bin_file_check", res1[0][0])
-        self.assertEqual(" Pre-checking phase ended on Tue Jun 15 12:12:58 UTC 2021 and took about 00:00:33",
+        self.assertEqual("Test ended on Sat Jul  3 00:07:10 UTC 2021 and took about 00:34:24",
                          res2[0][1])

embark/entrypoint.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 export DJANGO_SETTINGS_MODULE=embark.settings
-python3 manage.py runapscheduler &
+python3 manage.py migrate
+python3 manage.py runapscheduler --test &
 uwsgi --wsgi-file /app/embark/embark/wsgi.py --http :8000 --processes 2 --threads 10 &
 daphne embark.asgi:application -p 8001 -b '0.0.0.0'

embark/requirements.txt

@@ -3,14 +3,14 @@ uWSGI
 ipython
 django-dotenv
 pycodestyle
-channels
+channels_redis
+channels==3.0.3
 rx
 daphne
 inotify_simple
 mysqlclient
 pymongo
 mongoengine
-msgpack==0.6.2
-channels_redis==2.4.0
+msgpack
 django-apscheduler
-psutil
+psutil