Correctness checking for security_barrier_camera_demo w/ 1 network multi channels with inputting 1 image

demos/common/cpp/utils/include/utils/threads_common.hpp

@@ -100,7 +100,8 @@ class Worker {
         }
     }
     void stop() {
-        running = false;
+        if (tasks.empty())
+            running = false;
         tasksCondVar.notify_all();
     }
     void join() {

demos/common/cpp/utils/src/args_helper.cpp

@@ -81,7 +81,7 @@ std::vector<std::string> parseDevices(const std::string& device_string) {
     const std::string::size_type colon_position = device_string.find(":");
     if (colon_position != std::string::npos) {
         std::string device_type = device_string.substr(0, colon_position);
-        if (device_type == "HETERO" || device_type == "MULTI") {
+        if (device_type == "HETERO" || device_type == "MULTI" || device_type == "AUTO") {
             std::string comma_separated_devices = device_string.substr(colon_position + 1);
             std::vector<std::string> devices = split(comma_separated_devices, ',');
             for (auto& device : devices)

demos/face_recognition_demo/python/face_recognition_demo.py

@@ -42,7 +42,7 @@
 
 log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.DEBUG, stream=sys.stdout)
 
-DEVICE_KINDS = ['CPU', 'GPU', 'MYRIAD', 'HETERO', 'HDDL']
+DEVICE_KINDS = ['CPU', 'GPU', 'AUTO', 'AUTO:CPU,GPU', 'MULTI:CPU,GPU', 'MYRIAD', 'HETERO', 'HDDL']
 
 
 def build_argparser():

demos/security_barrier_camera_demo/cpp/main.cpp

@@ -220,8 +220,8 @@ class ClassifiersAggregator {
         std::mutex& printMutex = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context.classifiersAggregatorPrintMutex;
         printMutex.lock();
         if (FLAGS_r && !rawDetections.empty()) {
-            slog::debug << "Frame #: " << sharedVideoFrame->frameId << slog::endl;
-            slog::debug << rawDetections;
+            for (const std::string& rawDetection : rawDetections)
+                slog::debug << rawDetection << slog::endl;
             // destructor assures that none uses the container
             for (const std::string& rawAttribute : rawAttributes.container) {
                 slog::debug << rawAttribute << slog::endl;
@@ -292,6 +292,10 @@ ReborningVideoFrame::~ReborningVideoFrame() {
         context.videoFramesContext.lastFrameIdsMutexes[sourceID].lock();
         const auto frameId = ++context.videoFramesContext.lastframeIds[sourceID];
         context.videoFramesContext.lastFrameIdsMutexes[sourceID].unlock();
+
+        // Stop reborning when frame ID reached to input queue size
+        if (!context.isVideo && frameId >= FLAGS_n_iqs)
+            return;
         std::shared_ptr<ReborningVideoFrame> reborn = std::make_shared<ReborningVideoFrame>(context, sourceID, frameId, frame);
         worker->push(std::make_shared<Reader>(reborn));
     } catch (const std::bad_weak_ptr&) {}
@@ -379,7 +383,9 @@ void Drawer::process() {
         } else {
             if (!context.isVideo) {
                 try {
-                    std::shared_ptr<Worker>(context.drawersContext.drawersWorker)->stop();
+                    // Exit only when inferences on all of frames are finished.
+                    if (context.frameCounter >= FLAGS_n_iqs * context.readersContext.inputChannels.size())
+                        std::shared_ptr<Worker>(context.drawersContext.drawersWorker)->stop();
                 }
                 catch (const std::bad_weak_ptr&) {}
             }
@@ -428,7 +434,8 @@ bool DetectionsProcessor::isReady() {
     if (requireGettingNumberOfDetections) {
         classifiersAggregator = std::make_shared<ClassifiersAggregator>(sharedVideoFrame);
         std::list<Detector::Result> results;
-        results = context.inferTasksContext.detector.getResults(*inferRequest, sharedVideoFrame->frame.size(), classifiersAggregator->rawDetections);
+        results = context.inferTasksContext.detector.getResults(*inferRequest, sharedVideoFrame->sourceID, sharedVideoFrame->frameId, sharedVideoFrame->frame.size(), classifiersAggregator->rawDetections);
+
         for (Detector::Result result : results) {
             switch (result.label) {
                 case 1:
@@ -585,7 +592,6 @@ void InferTask::process() {
     std::reference_wrapper<ov::InferRequest> inferRequest = detectorsInfers.inferRequests.container.back();
     detectorsInfers.inferRequests.container.pop_back();
     detectorsInfers.inferRequests.mutex.unlock();
-
     context.inferTasksContext.detector.setImage(inferRequest, sharedVideoFrame->frame);
 
     inferRequest.get().set_callback(
@@ -607,6 +613,7 @@ void InferTask::process() {
 bool Reader::isReady() {
     Context& context = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context;
     context.readersContext.lastCapturedFrameIdsMutexes[sharedVideoFrame->sourceID].lock();
+    // Look for the next frame
     if (context.readersContext.lastCapturedFrameIds[sharedVideoFrame->sourceID] + 1 == sharedVideoFrame->frameId) {
         return true;
     } else {

demos/security_barrier_camera_demo/cpp/net_wrappers.hpp

@@ -121,9 +121,8 @@ class Detector {
         }
     }
 
-    std::list<Result> getResults(ov::InferRequest& inferRequest, cv::Size upscale, std::vector<std::string>& rawResults) {
-        // there is no big difference if InferReq of detector from another device is passed
-        // because the processing is the same for the same topology
+    std::list<Result> getResults(ov::InferRequest& inferRequest, const int64_t channelID, const int64_t frameID, cv::Size upscale, std::vector<std::string>& rawResults) {
+        // there is no big difference if InferReq of detector from another device is passed because the processing is the same for the same topology
         std::list<Result> results;
         ov::Tensor output_tensor = inferRequest.get_tensor(m_detectorOutputName);
         const float* const detections = output_tensor.data<float>();
@@ -146,8 +145,9 @@ class Detector {
             rect.height = static_cast<int>(detections[i * objectSize + 6] * upscale.height) - rect.y;
             results.push_back(Result{label, confidence, rect});
             std::ostringstream rawResultsStream;
-            rawResultsStream << "[" << i << "," << label << "] element, prob = " << confidence
-                        << "    (" << rect.x << "," << rect.y << ")-(" << rect.width << "," << rect.height << ")";
+            rawResultsStream << "ChannelId:" << channelID << ",FrameId:" << frameID << ",";
+            rawResultsStream << "ObjId:" << i << "," << label << "," << confidence
+                        << "," << rect.x << "," << rect.y << "," << rect.width << "," << rect.height;
             rawResults.push_back(rawResultsStream.str());
         }
         return results;

demos/tests/cases.py

@@ -591,10 +591,11 @@ def single_option_cases(key, *args):
             model_keys=['-m', '-m_lpr', '-m_va'],
             device_keys=['-d', '-d_lpr', '-d_va'],
             test_cases=combine_cases(
-        TestCase(options={'-no_show': None,
+        TestCase(options={'-no_show': None, '-r' : None, '-n_iqs': '1',
             **MONITORS,
             '-i': DataDirectoryArg('vehicle-license-plate-detection-barrier')}),
-        TestCase(options={'-m': ModelArg('vehicle-license-plate-detection-barrier-0106')}),
+        # Change the model in order to obtain the ROI data
+        TestCase(options={'-m': ModelArg('vehicle-license-plate-detection-barrier-0123')}),
         single_option_cases('-m_lpr',
             None,
             ModelArg('license-plate-recognition-barrier-0001'),

demos/tests/correctness_checker.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from unittest import case
+
+class Demo(ABC):
+    def __init__(self, name, implementation):
+        self.name = name
+        self.implementation = implementation
+        self.subdirectory = name + '/' + implementation
+        self.results = {}
+
+    @abstractmethod
+    def parser(self):
+        pass 
+
+    @abstractmethod
+    def checker(self):
+        pass
+
+class CPPDemo(Demo):
+    def __init__(self, name, implementation='cpp'):
+        super().__init__(name, implementation)
+        self.results = {}
+        pass
+
+    def parser(self):
+        fo = None
+        try:
+            fo = open('/tmp/' + self.subdirectory + '/results.log', 'r')
+        except IOError as err:
+            print("File error: " + str(err))
+        output = [i.rstrip() for i in fo.readlines()]
+        device = ''
+        case_index = ''
+        index = 1
+        while index < len(output):
+            if "Device" in output[index]:
+                device = output[index][output[index].find(":") + 1:]
+                if device not in self.results:
+                    self.results[device] = {}
+
+            if "CaseId" in output[index]:
+                case_index = output[index][output[index].find(":") + 1:]
+                if case_index not in self.results[device]:
+                    self.results[device][case_index] = {}
+
+            if "Execution_time" in output[index]:
+                execution_time = output[index].split(':')[1]
+                if execution_time == '-1':
+                   while index < len(output) and 'Device' not in output[index]:
+                       index += 1 
+                   continue
+
+            # Pase the raw data
+            while index < len(output) and 'ChannelId' in output[index]:
+                item = output[index][output[index].find('ChannelId'):].split(',')
+                # Channel ID
+                frame_results = {}
+                channel = item[0].split(':')
+                if channel[1] not in self.results[device][case_index]:
+                    self.results[device][case_index][channel[1]] = frame_results
+
+                # Frame ID
+                object_results = {}
+                frame = item[1].split(':')
+                if frame[1] not in self.results[device][case_index][channel[1]]:
+                    self.results[device][case_index][channel[1]][frame[1]] = object_results
+
+                # Object ID
+                label_prob_pos_results = []
+                objid = item[2].split(':')
+                if objid[1] not in self.results[device][case_index][channel[1]][frame[1]]:
+                    self.results[device][case_index][channel[1]][frame[1]][objid[1]] = label_prob_pos_results
+                self.results[device][case_index][channel[1]][frame[1]][objid[1]] = item[3:]
+                index += 1
+
+            index += 1
+
+
+    def checker(self):
+        self.parser()
+
+        flag = False
+        if "CPU" in self.results and "AUTO:CPU" in self.results:
+            if self.results['CPU'] == self.results['AUTO:CPU']:
+                flag = True
+            else:
+                print ("CPU vs AUTO:CPU have inconsistent results")
+
+        if "GPU" in self.results and "AUTO:GPU" in self.results:
+            if self.results['GPU'] == self.results['AUTO:GPU']:
+                flag = True
+            else:
+                print ("GPU vs AUTO:GPU have inconsistent results")
+
+        if not flag:
+            for device in self.results:
+                for case in self.results[device]:
+                    print ("---* Device: {} - Case: {} *----\n".format(device, case))
+                    for channel in self.results[device][case]:
+                        print ("Channel: {} - :{}".format(channel, self.results[device][case][channel]))
+                    print ('---------------------------------------------------------')
+        return flag
+
+
+
+DEMOS = [
+    CPPDemo(name='security_barrier_camera_demo')
+]
+def main():
+   for demo in DEMOS:
+       print ("Checking {}...".format(demo.name))
+       if demo.checker():
+           print("Demo: {} passed.".format(demo.name))
+       else:
+           print("Demo: {} failed.".format(demo.name))
+
+
+if __name__ == '__main__':
+    main()

demos/tests/run_tests.py

@@ -33,6 +33,7 @@
 import contextlib
 import csv
 import json
+from operator import contains
 import os
 import shlex
 import subprocess # nosec - disable B404:import-subprocess check
@@ -261,7 +262,12 @@ def main():
             print(header)
             print()
             demo.set_precisions(args.precisions, model_info)
-
+            filename = '/tmp/' + demo.subdirectory
+            os.makedirs(filename, exist_ok=True)
+            fo = open(filename + '/results.log', 'w+')
+            print("Save to {}".format(filename))
+            content = '' 
+            content += 'Testing {}...'.format(demo.subdirectory) + '\n'
             declared_model_names = set()
             for model_data in json.loads(subprocess.check_output(
                     [sys.executable, '--', str(auto_tools_dir / 'info_dumper.py'),
@@ -325,13 +331,20 @@ def option_to_args(key, value):
                             for arg in fixed_args + dev_arg + case_args))
                         print(test_descr)
                         print(flush=True)
+                        content += "Device:{}\nCaseId:{}\n".format(device, test_case_index)
+                        rawResults = '' 
+                        execution_time = -1 
                         try:
                             start_time = timeit.default_timer()
                             output = subprocess.check_output(fixed_args + dev_arg + case_args,
                                 stderr=subprocess.STDOUT, universal_newlines=True, encoding='utf-8',
                                 env=demo_environment, timeout=600)
                             execution_time = timeit.default_timer() - start_time
-                            demo.parse_output(output, test_case, device)
+                            for line in output.split('\n'):
+                                if "DEBUG" in line:
+                                    rawResults += line
+                                    rawResults += '\n'
+                            demo.parse_output(output, device, test_case_index)
                         except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
                             output = e.output
                             if isinstance(e, subprocess.CalledProcessError):
@@ -343,15 +356,20 @@ def option_to_args(key, value):
                             failed_tests.append(test_descr + '\n' + exit_msg)
                             num_failures += 1
                             execution_time = -1
-
+                            rawResults = {}
+
+                        content += "Execution_time:{}\n".format(execution_time)
+                        content += "{}\n".format(rawResults)
+                        fo.write(content)
+                        content = ''
                         if args.report_file:
                             collect_result(demo.subdirectory, device, case_model_names, execution_time, args.report_file)
                         if args.log_file:
                             if test_case_index == 0:
                                 write_log(header, args.log_file)
                             write_log(test_descr, args.log_file)
                             write_log(output, args.log_file)
-
+            fo.close()
             print()
 
     print("{} failures:".format(num_failures))