[Datumaro] Pip installation (#881)

* Add version file
* Remove unnecessary dependencies
* Add lxml use motivation
* Add pip setup script
* Reduce opencv dependency
* Fix cli command
* Codacy

datumaro/datumaro/__init__.py

@@ -23,7 +23,7 @@
     stats_command as stats_command_module,
     explain_command as explain_command_module,
 )
-from .components.config import VERSION
+from .version import VERSION
 
 
 KNOWN_COMMANDS = {

datumaro/datumaro/cli/explain_command.py

@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: MIT
 
 import argparse
-import cv2
 import logging as log
 import os
 import os.path as osp
@@ -13,7 +12,7 @@
 from datumaro.components.algorithms.rise import RISE
 from datumaro.util.command_targets import (TargetKinds, target_selector,
     ProjectTarget, SourceTarget, ImageTarget, is_project_path)
-from datumaro.util.image import load_image
+from datumaro.util.image import load_image, save_image
 from .util.project import load_project
 
 
@@ -60,6 +59,7 @@ def build_parser(parser=argparse.ArgumentParser()):
     return parser
 
 def explain_command(args):
+    import cv2
     from matplotlib import cm
 
     project = load_project(args.project_dir)
@@ -110,7 +110,7 @@ def explain_command(args):
             for j, heatmap in enumerate(heatmaps):
                 save_path = osp.join(args.save_dir,
                     file_name + '-heatmap-%s.png' % j)
-                cv2.imwrite(save_path, heatmap * 255.0)
+                save_image(save_path, heatmap * 255.0)
         else:
             for j, heatmap in enumerate(heatmaps):
                 disp = (image + cm.jet(heatmap)[:, :, 2::-1]) / 2
@@ -151,7 +151,7 @@ def explain_command(args):
                 for j, heatmap in enumerate(heatmaps):
                     save_path = osp.join(args.save_dir,
                         file_name + '-heatmap-%s.png' % j)
-                    cv2.imwrite(save_path, heatmap * 255.0)
+                    save_image(save_path, heatmap * 255.0)
 
             if args.progressive:
                 for j, heatmap in enumerate(heatmaps):

datumaro/datumaro/cli/project/diff.py

@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: MIT
 
 from collections import Counter
-import cv2
 from enum import Enum
 import numpy as np
 import os
@@ -19,6 +18,7 @@
     _formats.append('tensorboard')
 
 from datumaro.components.extractor import AnnotationType
+from datumaro.util.image import save_image
 
 
 Format = Enum('Formats', _formats)
@@ -135,8 +135,13 @@ def update_bbox_confusion(self, bbox_diff):
 
     @classmethod
     def draw_text_with_background(cls, frame, text, origin,
-            font=cv2.FONT_HERSHEY_SIMPLEX, scale=1.0,
+            font=None, scale=1.0,
             color=(0, 0, 0), thickness=1, bgcolor=(1, 1, 1)):
+        import cv2
+
+        if not font:
+            font = cv2.FONT_HERSHEY_SIMPLEX
+
         text_size, baseline = cv2.getTextSize(text, font, scale, thickness)
         cv2.rectangle(frame,
             tuple((origin + (0, baseline)).astype(int)),
@@ -148,6 +153,8 @@ def draw_text_with_background(cls, frame, text, origin,
         return text_size, baseline
 
     def draw_detection_roi(self, frame, x, y, w, h, label, conf, color):
+        import cv2
+
         cv2.rectangle(frame, (x, y), (x + w, y + h), color, 2)
 
         text = '%s %.2f%%' % (label, 100.0 * conf)
@@ -216,7 +223,7 @@ def save_item_bbox_diff(self, item_a, item_b, diff):
             path = osp.join(self.save_dir, 'diff_%s' % item_a.id)
 
             if self.output_format is Format.simple:
-                cv2.imwrite(path + '.png', img)
+                save_image(path + '.png', img)
             elif self.output_format is Format.tensorboard:
                 self.save_as_tensorboard(img, path)
 

datumaro/datumaro/components/algorithms/rise.py

@@ -5,7 +5,6 @@
 
 # pylint: disable=unused-variable
 
-import cv2
 import numpy as np
 from math import ceil
 
@@ -79,6 +78,8 @@ def normalize_hmaps(self, heatmaps, counts):
         return np.reshape(mhmaps, heatmaps.shape)
 
     def apply(self, image, progressive=False):
+        import cv2
+
         assert len(image.shape) == 3, \
             "Expected an input image in (H, W, C) format"
         assert image.shape[2] in [3, 4], \

datumaro/datumaro/components/config.py

@@ -234,5 +234,4 @@ def set(self, key, value):
             return super().set(key, value)
 
 
-VERSION = '0.1.0'
 DEFAULT_FORMAT = 'datumaro'

datumaro/datumaro/components/converters/datumaro.py

@@ -5,7 +5,6 @@
 
 # pylint: disable=no-self-use
 
-import cv2
 import json
 import os
 import os.path as osp
@@ -19,6 +18,7 @@
     LabelCategories, MaskCategories, PointsCategories
 )
 from datumaro.components.formats.datumaro import DatumaroPath
+from datumaro.util.image import save_image
 from datumaro.util.mask_tools import apply_colormap
 
 
@@ -133,7 +133,7 @@ def _save_mask(self, mask):
             DatumaroPath.MASKS_DIR)
         os.makedirs(masks_dir, exist_ok=True)
         path = osp.join(masks_dir, filename)
-        cv2.imwrite(path, mask)
+        save_image(path, mask)
         return mask_id
 
     def _convert_mask_object(self, obj):
@@ -279,7 +279,7 @@ def _save_image(self, item):
 
         image_path = osp.join(self._images_dir,
             str(item.id) + DatumaroPath.IMAGE_EXT)
-        cv2.imwrite(image_path, image)
+        save_image(image_path, image)
 
 class DatumaroConverter(Converter):
     def __init__(self, save_images=False, apply_colormap=False):

datumaro/datumaro/components/converters/ms_coco.py

@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: MIT
 
-import cv2
 import json
 import numpy as np
 import os
@@ -17,6 +16,7 @@
 )
 from datumaro.components.formats.ms_coco import CocoAnnotationType, CocoPath
 from datumaro.util import find
+from datumaro.util.image import save_image
 import datumaro.util.mask_tools as mask_tools
 
 
@@ -374,7 +374,7 @@ def make_task_converters(self):
 
     def save_image(self, item, filename):
         path = osp.join(self._images_dir, filename)
-        cv2.imwrite(path, item.image)
+        save_image(path, item.image)
 
         return path
 

datumaro/datumaro/components/converters/voc.py

@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: MIT
 
-import cv2
 from collections import OrderedDict, defaultdict
 import os
 import os.path as osp
@@ -14,6 +13,7 @@
 from datumaro.components.formats.voc import VocLabel, VocAction, \
     VocBodyPart, VocPose, VocTask, VocPath, VocColormap, VocInstColormap
 from datumaro.util import find
+from datumaro.util.image import save_image
 from datumaro.util.mask_tools import apply_colormap
 
 
@@ -111,7 +111,7 @@ def save_subsets(self):
                 if self._save_images:
                     data = item.image
                     if data is not None:
-                        cv2.imwrite(osp.join(self._images_dir,
+                        save_image(osp.join(self._images_dir,
                                 str(item_id) + VocPath.IMAGE_EXT),
                             data)
 
@@ -334,7 +334,7 @@ def save_segm(self, path, annotation, colormap):
             if colormap is None:
                 colormap = VocColormap
             data = apply_colormap(data, colormap)
-        cv2.imwrite(path, data)
+        save_image(path, data)
 
 class VocConverter(Converter):
     def __init__(self, task=None, save_images=False, apply_colormap=False):

datumaro/datumaro/components/dataset_filter.py

@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: MIT
 
-from lxml import etree as ET
+from lxml import etree as ET # NOTE: lxml has proper XPath implementation
 from datumaro.components.extractor import (DatasetItem, Annotation,
     LabelObject, MaskObject, PointsObject, PolygonObject,
     PolyLineObject, BboxObject, CaptionObject,

datumaro/datumaro/components/extractor.py

@@ -182,7 +182,8 @@ def __eq__(self, other):
             return False
         return \
             (self.label == other.label) and \
-            (np.all(self.image == other.image))
+            (self.image is not None and other.image is not None and \
+                np.all(self.image == other.image))
 
 def compute_iou(bbox_a, bbox_b):
     aX, aY, aW, aH = bbox_a
@@ -461,7 +462,9 @@ def __eq__(self, other):
             (self.id == other.id) and \
             (self.subset == other.subset) and \
             (self.annotations == other.annotations) and \
-            (self.image == other.image)
+            (self.has_image == other.has_image) and \
+            (self.has_image and np.all(self.image == other.image) or \
+                not self.has_image)
 
 class IExtractor:
     def __iter__(self):

datumaro/datumaro/components/launchers/openvino.py

@@ -5,7 +5,6 @@
 
 # pylint: disable=exec-used
 
-import cv2
 import os
 import os.path as osp
 import numpy as np
@@ -142,6 +141,8 @@ def _load_executable_net(self, batch_size=1):
         self._net = plugin.load(network=network, num_requests=1)
 
     def infer(self, inputs):
+        import cv2
+
         assert len(inputs.shape) == 4, \
             "Expected an input image in (N, H, W, C) format, got %s" % \
                 (inputs.shape)

datumaro/datumaro/util/command_targets.py

@@ -4,10 +4,10 @@
 # SPDX-License-Identifier: MIT
 
 import argparse
-import cv2
 from enum import Enum
 
 from datumaro.components.project import Project
+from datumaro.util.image import load_image
 
 
 TargetKinds = Enum('TargetKinds',
@@ -50,7 +50,10 @@ def is_inference_path(value):
     return False
 
 def is_image_path(value):
-    return cv2.imread(value) is not None
+    try:
+        return load_image(value) is not None
+    except Exception:
+        return False
 
 
 class Target:

datumaro/datumaro/util/image.py

@@ -3,23 +3,60 @@
 #
 # SPDX-License-Identifier: MIT
 
-import cv2
+# pylint: disable=unused-import
+
 import numpy as np
 
+from enum import Enum
+_IMAGE_BACKENDS = Enum('_IMAGE_BACKENDS', ['cv2', 'PIL'])
+_IMAGE_BACKEND = None
+try:
+    import cv2
+    _IMAGE_BACKEND = _IMAGE_BACKENDS.cv2
+except ModuleNotFoundError:
+    import PIL
+    _IMAGE_BACKEND = _IMAGE_BACKENDS.PIL
+
 from datumaro.util.image_cache import ImageCache as _ImageCache
 
 
 def load_image(path):
     """
     Reads an image in the HWC Grayscale/BGR(A) float [0; 255] format.
     """
-    image = cv2.imread(path)
-    image = image.astype(np.float32)
+
+    if _IMAGE_BACKEND == _IMAGE_BACKENDS.cv2:
+        import cv2
+        image = cv2.imread(path)
+        image = image.astype(np.float32)
+    elif _IMAGE_BACKEND == _IMAGE_BACKENDS.PIL:
+        from PIL import Image
+        image = Image.open(path)
+        image = np.asarray(image, dtype=np.float32)
+        if len(image.shape) == 3 and image.shape[2] in [3, 4]:
+            image[:, :, :3] = image[:, :, 2::-1] # RGB to BGR
+    else:
+        raise NotImplementedError()
 
     assert len(image.shape) == 3
     assert image.shape[2] in [1, 3, 4]
     return image
 
+def save_image(path, image):
+    if _IMAGE_BACKEND == _IMAGE_BACKENDS.cv2:
+        import cv2
+        cv2.imwrite(path, image)
+    elif _IMAGE_BACKEND == _IMAGE_BACKENDS.PIL:
+        from PIL import Image
+        image = image.astype(np.uint8)
+        if len(image.shape) == 3 and image.shape[2] in [3, 4]:
+            image[:, :, :3] = image[:, :, 2::-1] # BGR to RGB
+        image = Image.fromarray(image)
+        image.save(path)
+    else:
+        raise NotImplementedError()
+
+
 class lazy_image:
     def __init__(self, path, loader=load_image, cache=None):
         self.path = path

datumaro/datumaro/version.py

@@ -0,0 +1 @@
+VERSION = '0.1.0'

datumaro/requirements.txt

@@ -6,6 +6,4 @@ opencv-python>=4.1.0.25
 Pillow>=6.1.0
 pycocotools>=2.0.0
 PyYAML>=5.1.1
-requests>=2.20.0
-tensorboard>=1.12.0
 tensorboardX>=1.8