Add mapping from instance color to name

renderer/README.md

@@ -35,7 +35,7 @@ C++:
 ```
 ./test-rectangle.bin [egl/headless/glfw]		# a small tool to verify that rendering works
 ./objview.bin xx.obj	# viewer (require a display to show images)
-./objview-suncg.bin xx.obj ModelCategoryMapping.csv	 # viewer without person
+./objview-suncg.bin xx.obj ModelCategoryMapping.csv	 colormap_coarse.csv  # viewer in SUNCG mode
 ./objview-offline.bin xx.obj # render without display (to test its availability on server)
 ```
 
@@ -45,7 +45,10 @@ cd /path/to/House3DRepo/tests
 export PYTHONPATH=..
 python test-rendering.py /path/to/suncg/house/house.obj
 ```
-Check `test-rendering.py` for the API usage.
+Check `test-rendering.py` for the API usage. Read docstrings in 
+[suncg/render.hh](https://github.com/facebookresearch/House3D/blob/master/renderer/suncg/render.hh) 
+for detailed explanation on the APIs.
+
 Example data can be found at [releases](https://github.com/facebookresearch/House3D/releases/tag/example-data).
 
 ## Choosing the Rendering Backend:

renderer/python/pybind.cc

@@ -35,7 +35,9 @@ PYBIND11_MODULE(objrender, m) {
     .def("loadScene", &SUNCGRenderAPI::loadScene)
     .def("resolution", &SUNCGRenderAPI::resolution)
     .def("render", &SUNCGRenderAPI::render)
-    .def("renderCubeMap", &SUNCGRenderAPI::renderCubeMap);
+    .def("renderCubeMap", &SUNCGRenderAPI::renderCubeMap)
+    .def("getNameFromInstanceColor", &SUNCGRenderAPI::getNameFromInstanceColor)
+      ;
 
 
   py::class_<SUNCGRenderAPIThread>(m, "RenderAPIThread")
@@ -48,7 +50,9 @@ PYBIND11_MODULE(objrender, m) {
     .def("loadScene", &SUNCGRenderAPIThread::loadScene)
     .def("resolution", &SUNCGRenderAPIThread::resolution)
     .def("render", &SUNCGRenderAPIThread::render)
-    .def("renderCubeMap", &SUNCGRenderAPIThread::renderCubeMap);
+    .def("renderCubeMap", &SUNCGRenderAPIThread::renderCubeMap)
+    .def("getNameFromInstanceColor", &SUNCGRenderAPIThread::getNameFromInstanceColor)
+      ;
 
   auto camera = py::class_<Camera>(m, "Camera")
     .def("shift", &Camera::shift)

renderer/suncg/render.hh

@@ -57,8 +57,10 @@ class SUNCGRenderAPI {
     //  The mapping from color to class is in the CSV.
     //
     // For INSTANCE mode, returns a 3-channel image.
-    //  Each unique color means an instance and the coloring is consistent
-    //  across different views.
+    //  Each unique color means an instance, and the coloring of each instance
+    //  is consistent within a scene across different views.
+    //  You can use getNameFromInstanceColor(r, g, b) to know which object the
+    //  instance color maps to.
     //
     // For DEPTH mode, returns a 2-channel image:
     //  The first channel contains depth values scaled to (0, 255), where
@@ -95,7 +97,15 @@ class SUNCGRenderAPI {
     // Get the resolution.
     Geometry resolution() const { return geo_; }
 
-  private:
+    // r, g, b: integer in [0, 255]
+    // Returns: an object name defined in the obj file, or "" if not found.
+    // For SUNCG data, this object name is usually the "modelId" field
+    // in house.json.
+    std::string getNameFromInstanceColor(int r, int g, int b) {
+        return scene_->get_name_from_instance_color(r, g, b);
+    }
+
+    private:
     SceneCache scene_cache_;
     SUNCGScene* scene_ = nullptr; // no ownership
 
@@ -161,7 +171,11 @@ class SUNCGRenderAPIThread {
       });
     }
 
-  private:
+    std::string getNameFromInstanceColor(int r, int g, int b) {
+        return this->api_->getNameFromInstanceColor(r, g, b);
+    }
+
+    private:
     std::unique_ptr<SUNCGRenderAPI> api_;
     ExecutorInThread exec_;
 };

renderer/suncg/scene.cc

@@ -16,6 +16,7 @@ using namespace std;
 
 namespace {
 
+// Returns: a list of color, each represented as 3 integers in [0, 255]
 std::vector<glm::vec3> get_uniform_sampled_colors(int count) {
   int interval_length = static_cast<int>(pow(256, 3)) / (count + 2);
   int current_color = interval_length;
@@ -26,7 +27,7 @@ std::vector<glm::vec3> get_uniform_sampled_colors(int count) {
     int g = (current_color / 256) % 256;
     int b = (current_color / 256 / 256) % 256;
 
-    result.push_back(glm::vec3{r / 255.0, g / 255.0, b / 255.0});
+    result.push_back(glm::vec3{r, g, b});
 
     current_color += interval_length;
   }
@@ -215,6 +216,9 @@ void SUNCGScene::parse_scene() {
     auto& shp = obj_.shapes[i];
     glm::vec3 label_color = get_color_by_shape_name(shp.name);
     glm::vec3 instance_color = rand_instance_colors[shp.original_index];
+    int instance_color_key = (int)instance_color.x * 256 * 256 + (int)instance_color.y * 256 + (int)instance_color.z;
+    instance_color_to_name_[instance_color_key] = shp.name;
+    instance_color /= 255.;
     tinyobj::mesh_t& tmesh = shp.mesh;
     int nr_face = tmesh.num_face_vertices.size();
     auto& matids = tmesh.material_ids;

renderer/suncg/scene.hh

@@ -77,6 +77,11 @@ class SUNCGScene : public ObjSceneBase {
 
     RenderMode get_mode() const { return mode_; }
 
+    std::string get_name_from_instance_color(int r, int g, int b) {
+      int key = r * 256 * 256 + g * 256 + b;
+      return instance_color_to_name_[key];
+    }
+
   protected:
     void parse_scene();
 
@@ -113,6 +118,10 @@ class SUNCGScene : public ObjSceneBase {
     };
     // material for each mesh. Must have same size as mesh_
     std::vector<MaterialDesc> materials_;
+
+    // keys: r * 256 * 256 + g * 256 + b
+    // value: shape.name as in the obj file
+    std::unordered_map<int, std::string> instance_color_to_name_;
 };
 
 } // namespace render

tests/test-rendering.py

@@ -42,6 +42,10 @@
             mat = mat[:, :, ::-1]   # cv expects bgr
 
         if args.interactive:
+            if mode == RenderMode.INSTANCE:
+                center_rgb = mat[args.height // 2, args.width // 2, ::-1]
+                center_instance = api.getNameFromInstanceColor(center_rgb[0], center_rgb[1], center_rgb[2])
+                print("Instance ID in the center: ", center_instance)
             cv2.imshow("window", mat)
             key = cv2.waitKey(0)
             if key == 27 or key == ord('q'): #esc