ignore

.gitignore

@@ -1,3 +1,5 @@
+weights/
+.ipynb_checkpoints/
 data/
 .vscode/
 *.sublime-workspace

README.md

@@ -75,4 +75,6 @@ If you use find this code helpful, please consider citing:
     + [ ] how to create cached data
 - eval:
     + [ ] test time refinement
-    + [ ] predicted hand eval
+    + [ ] predicted hand eval
+    + [ ] add more demo images
+    + [ ] models

demo/demo_image.py

@@ -1,15 +1,3 @@
-"""
-Usage:
-python -m demo.demo_hoi -e xxx  --image xxx.png [--weight ....ckpt] [--out ]
-
-Usage:
-    # Run on image for bicycle class.
-    python demo.py --filename my_image.jpg --class_name bicycle
-    # Run on image without coarse interaction loss.
-    python demo.py --filename my_image.jpg --class_name bicycle --lw_inter 0
-    # Run on image with higher weight on depth ordering loss.
-    python demo.py --filename my_image.jpg --class_name bicycle --lw_depth 1.
-"""
 import argparse
 import logging
 import os.path as osp
@@ -27,6 +15,10 @@
 
 from nnutils.handmocap import get_handmocap_predictor, process_mocap_predictions, get_handmocap_detector
 from nnutils.hoiapi import get_hoi_predictor, vis_hand_object
+from nnutils import box2mask
+from detectron2.utils.visualizer import ColorMode
+from detectron2.utils.visualizer import Visualizer as VisDet
+from detectron2.structures.boxes import BoxMode
 
 
 def get_args():
@@ -35,12 +27,13 @@ def get_args():
         "--filename", default="demo/test.jpg", help="Path to image."
     )
     parser.add_argument("--out", default="output", help="Dir to save output.")
+    parser.add_argument("--view", default="ego_centric", help="Dir to save output.")
 
     parser.add_argument(
         "--experiment",
         "-e",
         dest="experiment_directory",
-        default='/glusterfs/yufeiy2/fair/ihoi_model/release_model/mow'
+        default='weights/mow'
     )
     parser.add_argument("opts",  default=None, nargs=argparse.REMAINDER)
 
@@ -55,11 +48,10 @@ def main(args):
     image = np.array(image)
     print(image.shape)
 
-    bbox_detector = get_handmocap_detector()
+    bbox_detector = get_handmocap_detector(args.view)
     # Process Human Estimations.
-    detect_output = bbox_detector.detect_hand_bbox(image[::-1].copy())
+    detect_output = bbox_detector.detect_hand_bbox(image[..., ::-1].copy())
     body_pose_list, body_bbox_list, hand_bbox_list, raw_hand_bboxes = detect_output
-
     res_img = visualizer.visualize(image, hand_bbox_list = hand_bbox_list)
     demo_utils.save_image(res_img, osp.join(args.out, 'hand_bbox.jpg'))
 
@@ -68,15 +60,23 @@ def main(args):
         image[..., ::-1], hand_bbox_list
     )
 
+    # MOW model also takes in masks but currently we feed in all 1. You could specify masks yourself, 
+    # or if you have bounding box for object masks, we can convert it to masks 
+
+    # mask_predictor = box2mask.setup_model()
+    # boxes = # object_bbox
+    # boxes = BoxMode.convert(boxes, BoxMode.XYWH_ABS, BoxMode.XYXY_ABS)
+    # predictions, object_mask = mask_predictor(image[..., ::-1], boxes, pad=0)
+    object_mask = np.ones_like(image[..., 0]) * 255
+
     hand_wrapper = ManopthWrapper().to('cpu')
     data = process_mocap_predictions(
-        mocap_predictions, image, hand_wrapper
+        mocap_predictions, image, hand_wrapper, mask=object_mask
     )
 
     hoi_predictor = get_hoi_predictor(args)
     output = hoi_predictor.forward_to_mesh(data)
-    vis_hand_object(output, data, image, args.out + '/test')
-
+    vis_hand_object(output, data, image, args.out + '/%s' % osp.basename(args.filename).split('.')[0])
 
 
 if __name__ == "__main__":

nnutils/box2mask.py

@@ -1,4 +1,6 @@
 from typing import List
+
+import numpy as np
 import torch
 from detectron2.modeling import build_model
 from detectron2.data import (
@@ -33,11 +35,11 @@ def __init__(self, cfg):
         self.input_format = cfg.INPUT.FORMAT
         assert self.input_format in ["RGB", "BGR"], self.input_format
 
-    def __call__(self, original_image, xyxy_boxes, is_object=None):
+    def __call__(self, original_image, xyxy_boxes, is_object=None, pad=0.1):
         """
         Args:
             original_image (np.ndarray): an image of shape (H, W, C) (in BGR order).
-
+            xyxy_boxes: (B, 4?)
         Returns:
             predictions (dict):
                 the output of the model for one image only.
@@ -52,14 +54,18 @@ def __call__(self, original_image, xyxy_boxes, is_object=None):
             image = self.aug.get_transform(original_image).apply_image(original_image)
             image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1))
             _, H, W = image.size()
-            print(image.size(), original_image.shape)
 
             if isinstance(xyxy_boxes, List):
                 xyxy_boxes = Boxes(xyxy_boxes)
-            xyxy_boxes = pad_box(xyxy_boxes, 0.1)
+            elif torch.is_tensor(xyxy_boxes):
+                xyxy_boxes = Boxes(xyxy_boxes)
+            xyxy_boxes = pad_box(xyxy_boxes, pad)
 
             if is_object is None:
                 is_object = torch.LongTensor([1, ] * len(xyxy_boxes))
+            if isinstance(is_object, List):
+                is_object = torch.FloatTensor(is_object)
+
             inputs = {"image": image, "height": height, "width": width}
 
             batched_inputs = [inputs]
@@ -83,15 +89,18 @@ def __call__(self, original_image, xyxy_boxes, is_object=None):
             predictions = model.roi_heads.box_predictor(box_features)  # (3, 81), (3, 320=80*4)
 
             # pred_classes: if is object class number, else person cat (0)
-            pred_classes = torch.argmax(predictions[0][:, :-1], 1) * is_object.to(device) # (3, )
+            pred_classes = torch.argmax(predictions[0][:, 1:-1], 1) * is_object.to(device) + 1 # (3, ) exclude person class [0]
             proposals.pred_boxes = proposals.proposal_boxes
             proposals.pred_classes = pred_classes
 
             # pred_instances: top(proposal * 80): pred_boxes, pred_classes, scores
             instances = model.roi_heads.forward_with_given_boxes(features, [proposals])
             predictions = GeneralizedRCNN._postprocess(instances, batched_inputs, images.image_sizes)[0]
 
-            return predictions
+            mask = predictions['instances'].pred_masks[0].cpu().detach().numpy()
+            mask = mask.astype(np.uint8) * 255
+            return predictions,   mask # [H, W]
+
 
 def pad_box(boxes: Boxes, ratio):
     box = boxes.tensor
@@ -113,7 +122,7 @@ def pad_box(boxes: Boxes, ratio):
 
 
 def setup_model() -> Box2Mask:
-    detbase='/private/home/yufeiy2/Tools//detectron2' # sys.argv[2]
+    detbase='externals/detectron2/' # sys.argv[2]
     cfg = get_cfg()
 
     point_rend.add_pointrend_config(cfg)

nnutils/handmocap.py

@@ -32,7 +32,7 @@ def get_handmocap_predictor(
 
 
 
-def process_mocap_predictions(mocap_predictions, image, hand_wrapper=None):
+def process_mocap_predictions(mocap_predictions, image, hand_wrapper=None, mask=None):
     if hand_wrapper is None:
         hand_wrapper = ManopthWrapper().to('cpu')
     one_hand = mocap_predictions[0]['right_hand']
@@ -54,7 +54,13 @@ def process_mocap_predictions(mocap_predictions, image, hand_wrapper=None):
     crop = image_utils.crop_resize(image, hoi_bbox, return_np=False)
     crop = ToTensor()(crop)[None] * 2 - 1
 
-    mask = torch.ones([1, 1, crop.shape[-2], crop.shape[-1]])
+    if mask is None:
+        mask = torch.ones([1, 1, crop.shape[-2], crop.shape[-1]])
+    else:
+        mask = image_utils.crop_resize(mask, hoi_bbox, return_np=False)
+        mask = ToTensor()(mask)[None]
+        print(mask.shape)
+
 
     data = {
         'cTh': geom_utils.matrix_to_se3(cTh),

nnutils/hoiapi.py

@@ -11,6 +11,7 @@
 from nnutils.hand_utils import ManopthWrapper, get_nTh
 
 
+
 def get_hoi_predictor(args):
     cfg_def = get_cfg_defaults()
     cfg_def = OmegaConf.create(cfg_def.dump())
@@ -85,7 +86,8 @@ def vis_hand_object(output, data, image, save_dir):
     cameras = PerspectiveCameras(cam_f, cam_p, device=device)
     iHoi = mesh_utils.render_mesh(cHoi, cameras,)
     image_utils.save_images(iHoi['image'], save_dir + '_cHoi', bg=data['image']/2+0.5, mask=iHoi['mask'])
-
+    image_utils.save_images(data['image']/2+0.5, save_dir + '_inp')
+
     image_list = mesh_utils.render_geom_rot(cHoi, cameras=cameras, view_centric=True)
     image_utils.save_gif(image_list, save_dir + '_cHoi')
 

scripts/install_third_party.sh

@@ -1,12 +1,21 @@
 set -x 
 
 # Install Frankmocap
-rm -r external/frankmocap
-mkdir -p external
+rm -r externals/frankmocap
+mkdir -p externals
 git clone https://github.com/facebookresearch/frankmocap.git external/frankmocap
-cd external/frankmocap
+cd externals/frankmocap
 bash scripts/install_frankmocap.sh
+cd ../..
 
 # install manopth
 pip install "git+https://github.com/hassony2/manopth.git"
 
+
+# install detectron2
+# clone the repo in order to access pre-defined configs in PointRend project
+cd externals
+!git clone --branch v0.6 https://github.com/facebookresearch/detectron2.git detectron2
+# install detectron2 from source
+!pip install -e detectron2
+# See https://detectron2.readthedocs.io/tutorials/install.html for other installation options