Merges BD V3.3 changes: Upgrading BuildingDepot to Python3

.github/workflows/test_bd.yml

@@ -2,35 +2,34 @@
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
 name: BuildingDepot
-
 on:
   push:
-    branches: 
-        - master
-        - develop
+    branches:
+      - master
+      - develop
   pull_request:
     branches:
-        - master
-        - develop
+      - develop
 jobs:
   build:
-
-    runs-on: ubuntu-18.04
-
+    runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 2.7
-      uses: actions/setup-python@v1
-      with:
-        python-version: 2.7
-    - name: Install BuildingDepot
-      run: |
-        sudo bash -x ./script_for_github_actions.sh
-    - name: Initialize tests
-      run: |
-        cd benchmarking-tools/functional-testing-tool
-        npm install
-    - name: Run tests
-      run: |
-        cd benchmarking-tools/functional-testing-tool
-        npm test ./tests/all.js
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          architecture: x64
+      - name: Display Python version
+        run: python --version
+      - name: Install BuildingDepot
+        run: |
+          sudo bash -x ./script_for_github_actions.sh
+      - name: Initialize tests
+        run: |
+          cd benchmarking-tools/functional-testing-tool
+          npm install
+      - name: Run tests
+        run: |
+          cd benchmarking-tools/functional-testing-tool
+          npm test ./tests/all.js

README.md

@@ -1,4 +1,4 @@
-BuildingDepot v3.2.9 ([link](https://buildingdepot.org/))
+BuildingDepot v3.3 ([link](https://buildingdepot.org/))
 ====================
 
 ![BuildingDepot](https://github.com/synergylabs/BuildingDepot-v3/workflows/BuildingDepot/badge.svg)
@@ -89,12 +89,12 @@ What's installed
 
 * The following packages are installed using apt-get
  * openssl
- * python-setuptools
- * python-dev
+ * python3-setuptools
+ * python3-dev
  * build-essential
- * python-software-properties
+ * python3-software-properties
  * mongodb
- * python-pip
+ * python3-pip
  * nginx
  * supervisor
  * redis-server
@@ -103,15 +103,18 @@ What's installed
 * The following packages are installed in the python virtual environment
  * Flask
  * mongoengine
- * flask-restful
- * Flask-HTTPAuth
- * flask-login
- * validate-email
- * requests
+ * Flask-Login
  * Flask-Script
+ * Flask-OAuthlib
+ * jsonschema
+ * pika
+ * Sphinx
+ * sphinx-theme
  * Flask-WTF
- * flask-bootstrap
+ * Flask-Bootstrap
+ * uWSGI
  * redis
  * influxdb
  * pymongo
+ * aniso8601
  * firebase-admin

benchmarking-tools/BD-Performance-Test/config.json

@@ -1,19 +1,19 @@
 {
-	"uri":{
-		"cs":"https://localhost:81",
-		"ds":"https://localhost:82"
-	},
-	"credentials":{
-		"clientId" : "",
-		"clientSecret" : ""
-	},
-	"defaults":{
-		"oauth":true,
-		"connections":1,
-		"amount":1
-	},
-	"testSuite":"./testSuites/post-time-series.json",
-	"headers":{
-		"content-type":"application/json"
-	}
+  "uri": {
+    "cs": "https://localhost:81",
+    "ds": "https://localhost:82"
+  },
+  "credentials": {
+    "clientId": "",
+    "clientSecret": ""
+  },
+  "defaults": {
+    "oauth": true,
+    "connections": 1,
+    "amount": 1
+  },
+  "testSuite": "./testSuites/post-time-series.json",
+  "headers": {
+    "content-type": "application/json"
+  }
 }

benchmarking-tools/BD-Performance-Test/lib/performanceTest.js

@@ -15,8 +15,7 @@ let addTestSuite = async function (testSuite) {
     let updateInterval = await oAuthHandler.handle();
     if (testSuite.delay !== undefined) {
         await delay.inMilliseconds(testSuite.delay)
-    }
-    else if (config.defaults.delay !== undefined) {
+    } else if (config.defaults.delay !== undefined) {
         await delay.inMilliseconds(config.defaults.delay)
     }
     for (let test of testSuite.tests) {
@@ -26,8 +25,7 @@ let addTestSuite = async function (testSuite) {
         let options = await testOptions.create(test);
         if (test.test === "create sensors for performance-testing") {
             await createSensors.runTest(options);
-        }
-        else {
+        } else {
             await runTest(options);
         }
     }
@@ -39,15 +37,14 @@ let addTestSuite = async function (testSuite) {
  * Create test suite.
  */
 let createTestSuite = async function (test) {
-    if(await testOptions.testExists(test)){
+    if (await testOptions.testExists(test)) {
         return {
             title: test,
             tests: [{
                 title: test
             }]
         }
-    }
-    else{
+    } else {
         console.log('Invalid testSuite or test!');
         process.exit(1);
     }
@@ -65,7 +62,7 @@ let runTest = async function (options) {
                 resolve(true)
             }
         };
-        if(options.title === undefined){
+        if (options.title === undefined) {
             options.title = options.test;
         }
         console.log('\nTitle: ' + options.title + '\n'.padEnd(options.title.length + 8, '#'));

benchmarking-tools/BD-Performance-Test/lib/testOptions.js

@@ -16,7 +16,7 @@ let formOptions = async function (test) {
     if (test.duration !== undefined) {
         delete options.amount
     }
-    if(await isInvalidUri(options.uri)){
+    if (await isInvalidUri(options.uri)) {
         options.url = uri[request.url];
     }
     if (request.idReplacement !== undefined)
@@ -25,32 +25,27 @@ let formOptions = async function (test) {
         request.body = await dataGenerator.formData(options.formData.saveFile, options.formData.totalSensors, options.formData.totalSamples, options.formData.totalValues, options.formData.dynamic);
         options['totalSensors'] = options.formData.totalSensors;
         options['totalSamples'] = options.formData.totalSamples;
-    }
-    else if (request.formData !== undefined) {
+    } else if (request.formData !== undefined) {
         request.body = await dataGenerator.formData(request.formData.saveFile, request.formData.totalSensors, request.formData.totalSamples, request.formData.totalValues, request.formData.dynamic);
         options['totalSensors'] = request.formData.totalSensors;
         options['totalSamples'] = request.formData.totalSamples;
-    }
-    else if (request.file !== undefined) {
+    } else if (request.file !== undefined) {
         request.body = (test.file === undefined) ? (await fs.readFile(request.file)).toString() : (await fs.readFile(test.file)).toString();
-    }
-    else if (request.body !== undefined) {
+    } else if (request.body !== undefined) {
         request.body = (test.body === undefined) ? JSON.stringify(request.body) : JSON.stringify(test.body);
-    }
-    else if (request.json !== undefined) {
+    } else if (request.json !== undefined) {
         request.body = (test.json === undefined) ? request.json : test.json;
     }
     if (options.headers === undefined) {
         options['headers'] = headers;
-    }
-    else {
+    } else {
         Object.assign(options.headers, headers);
     }
     if (test.title !== undefined) {
         options['title'] = test.title;
     }
     options['requests'] = [request];
-    if(options.amount !== undefined && options.connections > options.amount){
+    if (options.amount !== undefined && options.connections > options.amount) {
         console.log("Number of requests can't be less than number of concurrent connections.")
         process.exit(1)
     }
@@ -70,20 +65,20 @@ let formOptions = async function (test) {
  */
 let testExists = async function (test) {
     let allTests = Object.keys(tests);
-    return (allTests.indexOf(test)>-1)
+    return (allTests.indexOf(test) > -1)
 };
 
 /**
  * A function to check if the uri is valid
  * @return {boolean} Returns true if uri is invalid, false otherwise.
  */
 let isInvalidUri = async function (uri) {
-    let pattern = new RegExp('^(https?:\\/\\/)?'+ // protocol
-        '((([a-z\\d]([a-z\\d-]*[a-z\\d])*)\\.)+[a-z]{2,}|'+ // domain name
-        '((\\d{1,3}\\.){3}\\d{1,3}))'+ // OR ip (v4) address
-        '(\\:\\d+)?(\\/[-a-z\\d%_.~+]*)*'+ // port and path
-        '(\\?[;&a-z\\d%_.~+=-]*)?'+ // query string
-        '(\\#[-a-z\\d_]*)?$','i'); // fragment locator
+    let pattern = new RegExp('^(https?:\\/\\/)?' + // protocol
+        '((([a-z\\d]([a-z\\d-]*[a-z\\d])*)\\.)+[a-z]{2,}|' + // domain name
+        '((\\d{1,3}\\.){3}\\d{1,3}))' + // OR ip (v4) address
+        '(\\:\\d+)?(\\/[-a-z\\d%_.~+]*)*' + // port and path
+        '(\\?[;&a-z\\d%_.~+=-]*)?' + // query string
+        '(\\#[-a-z\\d_]*)?$', 'i'); // fragment locator
     return !pattern.test(uri);
 };
 

benchmarking-tools/BD-Performance-Test/lib/testResultsHandler.js

@@ -8,7 +8,7 @@ let handleTestResults = async function (res, options) {
     res.requests['average2xx'] = (res.requests.total - res.non2xx) / res.duration;
     res['pointPerSecond'] = res.requests['average2xx'] * options.totalSensors * options.totalSamples;
     console.log('Total Requests: ' + res.requests.total + ' requests\nAverage RPS: ' + res.requests.average2xx + ' req/s\np99 Latency: ' + res.latency.p99 + ' ms\nTotal throughput: ' + res.throughput.total + ' Bytes\nAverage throughput: ' + res.throughput.average + ' Bytes/sec\nConnections: ' + res.connections + '\nDuration: ' + res.duration + ' seconds\nNon-2xx: ' + res.non2xx);
-    if(res.pointPerSecond){
+    if (res.pointPerSecond) {
         console.log('Sensors: ' + options.totalSensors + '\nSamples: ' + options.totalSamples + '\nEffective points/sec: ' + res.pointPerSecond.toFixed(2) + ' points/sec written');
     }
     console.log('\n\n')

benchmarking-tools/BD-Performance-Test/results/parse.py

@@ -1,12 +1,23 @@
-from __future__ import print_function
-from os import listdir
 import json
-for f in listdir('.'):
-	if f == 'parse.py':
-		continue
-	with open('./'+f) as d:
-		try:
-			data = json.load(d)
-			print(data['title'] , data['requests']['total'] , data['requests']['average'] , data['latency']['p99'] , data['throughput']['total'] , data['throughput']['average'] , data['connections'] , data['duration'] , data['non2xx'], sep='\t')   
-		except Exception as e:
-			pass
+from os import listdir
+
+for f in listdir("."):
+    if f == "parse.py":
+        continue
+    with open("./" + f) as d:
+        try:
+            data = json.load(d)
+            print(
+                data["title"],
+                data["requests"]["total"],
+                data["requests"]["average"],
+                data["latency"]["p99"],
+                data["throughput"]["total"],
+                data["throughput"]["average"],
+                data["connections"],
+                data["duration"],
+                data["non2xx"],
+                sep="\t",
+            )
+        except Exception as e:
+            pass

benchmarking-tools/BD-Performance-Test/run.js

@@ -6,17 +6,15 @@ let config = require('./config.json'),
 
 if (process.argv[2] === undefined) {
     testSuite = config.testSuite;
-}
-else {
+} else {
     testSuite = process.argv[2];
 }
 
-let runTestSuite = async function(testSuite){
-    if(await fileSystemHandler.exists(testSuite)){
+let runTestSuite = async function (testSuite) {
+    if (await fileSystemHandler.exists(testSuite)) {
         let testSuiteJSON = (await fileSystemHandler.readFile(testSuite)).toString();
         run.addTestSuite(JSON.parse(testSuiteJSON));
-    }
-    else{
+    } else {
         run.addTestSuite(await run.createTestSuite(testSuite));
     }
 };

benchmarking-tools/BD-Performance-Test/test-suite.json

@@ -1,26 +1,28 @@
-{	"title":"Different number of sensors:",
-	"defaults": {
-		"test": "post time-series data"
-	},
-	"tests": [
-		{
-			"title": "Post Time-series : 1000 sensors",
-			"formData": {
-				"saveFile": "log.json",
-				"totalSensors": 1000,
-				"totalSamples": 1,
-				"totalValues": 1,
-				"dynamic": true
-			}
-		},
-		{
-			"title": "Post Time-series : 1000 sensors",
-			"formData": {
-				"saveFile": "log.json",
-				"totalSensors": 1000,
-				"totalSamples": 1,
-				"totalValues": 1,
-				"dynamic": false
-			}
-		}]
+{
+  "title": "Different number of sensors:",
+  "defaults": {
+    "test": "post time-series data"
+  },
+  "tests": [
+    {
+      "title": "Post Time-series : 1000 sensors",
+      "formData": {
+        "saveFile": "log.json",
+        "totalSensors": 1000,
+        "totalSamples": 1,
+        "totalValues": 1,
+        "dynamic": true
+      }
+    },
+    {
+      "title": "Post Time-series : 1000 sensors",
+      "formData": {
+        "saveFile": "log.json",
+        "totalSensors": 1000,
+        "totalSamples": 1,
+        "totalValues": 1,
+        "dynamic": false
+      }
+    }
+  ]
 }