Merge develop to master

.github/workflows/test_packages.yml

@@ -43,6 +43,7 @@ jobs:
           - perception/semantic_segmentation
           - perception/object_tracking_2d
           # - perception/object_tracking_3d # passes, but disabled due to free() crash
+          - perception/object_tracking_3d/single_object_tracking
           - perception/object_detection_2d/centernet
           - perception/object_detection_2d/detr
           - perception/object_detection_2d/gem

.github/workflows/tests_suite.yml

@@ -112,6 +112,7 @@ jobs:
           - perception/binary_high_resolution
           - perception/object_tracking_2d
           # - perception/object_tracking_3d # passes, but disabled due to free() crash
+          - perception/object_tracking_3d/single_object_tracking
           - perception/object_detection_2d/centernet
           - perception/object_detection_2d/detr
           - perception/object_detection_2d/gem

.github/workflows/tests_suite_develop.yml

@@ -114,6 +114,7 @@ jobs:
           - perception/binary_high_resolution
           - perception/object_tracking_2d
           # - perception/object_tracking_3d # passes, but disabled due to free() crash
+          - perception/object_tracking_3d/single_object_tracking
           - perception/object_detection_2d/centernet
           - perception/object_detection_2d/detr
           - perception/object_detection_2d/gem

.gitignore

@@ -47,6 +47,8 @@ Thumbs.db
 
 # Log files
 *.log
+*.txt
+!requirements.txt
 
 # IDE files
 .vscode

Makefile

@@ -1,4 +1,4 @@
-# Copyright 2020-2021 OpenDR European Project
+# Copyright 2020-2023 OpenDR European Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

dependencies/install.sh

@@ -20,11 +20,11 @@ if [ -f "linux_dependencies.txt" ]; then
        rm linux_dependencies.txt
 fi
 if [ -f "python_prerequisites.txt" ]; then
-       pip install -r python_prerequisites.txt
+       cat python_prerequisites.txt | while read PACKAGE; do pip install $PACKAGE; done
        rm python_prerequisites.txt
 fi
 if [ -f "python_dependencies.txt" ]; then
-       pip install -r python_dependencies.txt
+       cat python_dependencies.txt | while read PACKAGE; do pip install $PACKAGE; done
        rm python_dependencies.txt
 fi
 
@@ -35,10 +35,10 @@ if [ -f "linux_dependencies.txt" ]; then
        rm linux_dependencies.txt
 fi
 if [ -f "python_prerequisites.txt" ]; then
-       pip install -r python_prerequisites.txt
+       cat python_prerequisites.txt | while read PACKAGE; do pip install $PACKAGE; done
        rm python_prerequisites.txt
 fi
 if [ -f "python_dependencies.txt" ]; then
-       pip install -r python_dependencies.txt
+       cat python_dependencies.txt | while read PACKAGE; do pip install $PACKAGE; done
        rm python_dependencies.txt
 fi

dependencies/parse_dependencies.py

@@ -20,6 +20,7 @@
 import os
 import sys
 from configparser import ConfigParser
+from warnings import warn
 
 global flag_efficientNet
 flag_efficientNet = ''
@@ -28,10 +29,12 @@
 python_file = "python_dependencies.txt"
 linux_file = "linux_dependencies.txt"
 
+
 def read_ini(path):
     opendr_device = os.environ.get('OPENDR_DEVICE', default='cpu')
     parser = ConfigParser()
     parser.read(path)
+
     def read_ini_key(key, summary_file):
         tool_device = parser.get('device', 'opendr_device', fallback='cpu')
         if opendr_device == 'cpu' and tool_device == 'gpu':
@@ -45,6 +48,7 @@ def read_ini_key(key, summary_file):
                         continue
                     with open(summary_file, "a") as f:
                         f.write(os.path.expandvars(package) + '\n')
+
     read_ini_key('python-dependencies', python_prerequisites_file)
     read_ini_key('python', python_file)
     read_ini_key('linux', linux_file)
@@ -58,7 +62,106 @@ def efficientNetTrick(package):
     # only with EfficientLPS version
     elif 'EfficientPS' in package and 'EfficientLPS' not in flag_efficientNet:
         flag_efficientNet = package
-
+
+
+def verbose_print(message, verbose):
+    if verbose:
+        print(message)
+
+
+def add_dep_to_dict(dep_dict, dep, new_constraint, verbose=False):
+    if dep not in dep_dict.keys():
+        dep_dict[dep] = new_constraint
+        verbose_print(f"no issues adding new dependency: {dep}, {new_constraint}", verbose)
+        return
+    elif dep in dep_dict.keys() and dep_dict[dep] != new_constraint:
+        verbose_print(f"key already exists: {dep} as {dep_dict[dep]}, " +
+                      f"trying to re-add with new_constraint {new_constraint}", verbose)
+
+        # Cases where old/new constraint is loose/strict(==) are clear
+        if new_constraint is None:
+            verbose_print("Original is strict, not re-adding as loose, skipping...", verbose)
+        elif dep_dict[dep] is None:
+            verbose_print("Original is loose, re-adding with more strict, adding...", verbose)
+            dep_dict[dep] = new_constraint
+
+        # Cases where one of the dependencies is strict and the other is half-strict (>, <, >=, <=) are not that clear
+        elif "==" in new_constraint and "==" not in dep_dict[dep]:
+            verbose_print("Original is half-strict, re-adding with strict, adding...", verbose)
+            if dep_dict[dep][0] != new_constraint[0]:
+                warn("[WARNING] Filtering dependencies, probable clash of strict and non-strict versions "
+                     f"of same package. {dep} tries to constraint both to version {dep_dict[dep]} and "
+                     f"{new_constraint}, please review.")
+            dep_dict[dep] = new_constraint
+        elif "==" in dep_dict[dep] and "==" not in new_constraint:
+            verbose_print("Original is strict (==), new isn't, skipping...", verbose)
+            if dep_dict[dep][0] != new_constraint[0]:
+                warn("[WARNING] Filtering dependencies, probable clash of strict and non-strict versions "
+                     f"of same package. {dep} tries to constraint both to version {dep_dict[dep]} and "
+                     f"{new_constraint}, please review.")
+
+        # Critical clash of strict but different constraints
+        elif "==" in dep_dict[dep] and "==" in new_constraint:
+            raise EnvironmentError(f"Filtering dependencies, critical clash of strict versions of same package. "
+                                   f"{dep} tries to constraint both to version {dep_dict[dep]} and {new_constraint}.")
+
+        # Remaining cases with half-strict constraints, not that clear
+        else:
+            warn("[WARNING] Probably case where different half-strict (<=, >=, etc.) constraints clash. "
+                 f"Attempted to add {dep}, {new_constraint}. Kept original dependency {dep}, {dep_dict[dep]}, "
+                 "please review.")
+        return
+    else:
+        verbose_print(f"key already exists: {dep}, as  {dep_dict[dep]}, " +
+                      f"tried to add with same constraint {new_constraint}", verbose)
+        return
+
+
+def filter_dependencies(dependencies, verbose=False):
+    filt_dependencies = {}
+    filtered_dependencies_list = []
+
+    # Filter dependencies by adding them in a dictionary
+    for d in dependencies:
+        if "git" in d:
+            # Git dependency, add as-is
+            add_dep_to_dict(filt_dependencies, d, None, verbose)
+        elif "==" in d:
+            d, dep_ver = d.split("==")[0], d.split("==")[1]
+            add_dep_to_dict(filt_dependencies, d, [dep_ver, "=="], verbose)
+        elif "," in d:
+            # Upper and lower constraint, add as-is
+            add_dep_to_dict(filt_dependencies, d, None, verbose)
+            warn(f"[WARNING] added a dependency {d}, with both an upper and a lower version, please review.")
+        elif "<=" in d:
+            d, dep_ver = d.split("<=")[0], d.split("<=")[1]
+            add_dep_to_dict(filt_dependencies, d, [dep_ver, "<="], verbose)
+        elif ">=" in d:
+            d, dep_ver = d.split(">=")[0], d.split(">=")[1]
+            add_dep_to_dict(filt_dependencies, d, [dep_ver, ">="], verbose)
+        elif ">" in d:
+            d, dep_ver = d.split(">")[0], d.split(">")[1]
+            add_dep_to_dict(filt_dependencies, d, [dep_ver, ">"], verbose)
+        elif "<" in d:
+            d, dep_ver = d.split("<")[0], d.split("<")[1]
+            add_dep_to_dict(filt_dependencies, d, [dep_ver, "<"], verbose)
+        else:
+            # Simple dependency (no constraint), add as-is
+            add_dep_to_dict(filt_dependencies, d, None, verbose)
+        if verbose:
+            print("")
+
+    # Gather dependencies with their constraints in a list
+    for key, val in filt_dependencies.items():
+        full_dep = key
+        if val is not None:
+            full_dep += val[1] + val[0]
+        filtered_dependencies_list.append(full_dep)
+    if filtered_dependencies_list[-1] == "":
+        filtered_dependencies_list = filtered_dependencies_list[0:-1]  # Remove empty last line
+
+    return filtered_dependencies_list
+
 
 # Parse arguments
 section = "runtime"
@@ -88,3 +191,25 @@ def efficientNetTrick(package):
                     read_ini(os.path.join(subdir, filename))
 with open(python_file, "a") as f:
     f.write(os.path.expandvars(flag_efficientNet) + '\n')
+
+# Filter python dependencies
+python_dependencies = []
+with open(python_file, "r") as f:
+    for line in f:
+        # Grab line and remove newline char
+        python_dependencies.append(line.replace("\n", ""))
+
+# Set verbose to True to debug dependencies in detail
+filtered_dependencies = filter_dependencies(python_dependencies, verbose=False)
+print(f"{len(python_dependencies)} dependencies filtered down to {len(filtered_dependencies)}")
+
+# # Make igibson last in the installation order, which can fix possible igibson installation error
+# filtered_dependencies.remove("igibson==2.0.3")
+# filtered_dependencies.append("igibson==2.0.3")
+
+with open(python_file, "w") as f:
+    for i in range(len(filtered_dependencies)):
+        if i < len(filtered_dependencies) - 1:
+            f.write(filtered_dependencies[i] + "\n")
+        else:
+            f.write(filtered_dependencies[i])