Merge branch 'develop'

docs/CHANGELOG.md

@@ -85,3 +85,7 @@ v1.6.0
 - Add packaged font files to assets/fonts, fix too-many-open-fonts crash
 - Fix fish school disappearing at last frame in video
 - Fix crash from `fabrics.apply`
+
+v1.7.0
+- Implement camera IMU calculation and export
+- Add point tracking ground truth

docs/GroundTruthAnnotations.md

@@ -137,7 +137,17 @@ The mask of occluded pixels for the aforementioned optical flow is stored as a H
 
 *Note: This mask is computed by comparing the face-ids on the triangle meshes at either end of each flow vector. Infinigen meshes often contain multiple faces per-pixel, resulting in frequent false-negatives (negative=occluded). These false-negatives are generally distributed uniformly over the image (like salt-and-pepper noise), and can be reduced by max-pooling the occlusion mask down to the image resolution.*
 
-### Camera Intrinsics & Extrinsics
+### Point Trajectory & Point Trajectory Occlusion :large_blue_diamond:
+It is similar to Optical Flow and its Occlusion, except that it computes the flow from a specified source frame (the first frame by default) to all the other frames. Customize it by `--pipeline_overrides iterate_scene_tasks.point_trajectory_src_frame=X`.
+
+To visualize point trajectories for a scene, run:
+```
+python infinigen/tools/results/visualize_traj.py path/to/your/scene/frames
+```
+
+### Camera Pose
+
+##### Intrinsic & Extrinsic Matrices
 
 Camera intrinsics and extrinsics are stored as a numpy ".npz" file inside the "camview" folder.
 
@@ -147,6 +157,27 @@ The camera pose is stored as a 4 x 4 numpy matrix mapping from camera coordinate
 
 As is standard in computer vision, the assumed world coordinate system in the saved camera poses is +X -> Right, +Y -> Down, +Z Forward. This is opposed to how Blender internally represents geometry, with flipped Y and Z axes.
 
+##### Camera Velocity / Acceleration (IMU)
+
+Camera velocity / acceleration is stored as ".txt" files inside the "frames/imu_tum" folder.
+
+Inertial Measurement Unit (IMU) records the rotational velocity and linear acceleration of each camera at each frame. Each row contains the timestamp, rotational velocity (x y z), and linear acceleration (x y z) with each value separated by a space:
+
+```
+timestamp rv_x rv_y rv_z a_x a_y a_z
+```
+
+##### TUM Trajectory Format
+
+For convenience, we also save the trajectories of any animated cameras in the [TUM trajectory format](https://github.com/MichaelGrupp/evo/wiki/Formats#tum---tum-rgb-d-dataset-trajectory-format).
+
+
+Each row contains the timestamp, position, and rotation (as quarternion) with each value separated by a space:
+
+```
+timestamp x y z q_x q_y q_z q_w
+```
+
 ### Panoptic Segmentation
 
 Infinigen saves three types of semantic segmentation masks: 1) Object Segmentation 2) Tag Segmentation 3) Instance Segmentation

infinigen/__init__.py

@@ -6,7 +6,7 @@
 import logging
 from pathlib import Path
 
-__version__ = "1.6.1"
+__version__ = "1.7.0"
 
 
 def repo_root():

infinigen/core/execute_tasks.py

@@ -62,22 +62,70 @@ def render(
         render_image_func(frames_folder=Path(output_folder), camera_id=camera_id)
 
 
+def is_static(obj):
+    while True:
+        if obj.name.startswith("scatter:"):
+            return False
+        if obj.users_collection[0].name.startswith("assets:"):
+            return False
+        if obj.constraints is not None and len(obj.constraints) > 0:
+            return False
+        if obj.animation_data is not None:
+            return False
+        for modifier in obj.modifiers:
+            if modifier.type == "NODES":
+                if modifier.node_group.animation_data is not None:
+                    return False
+            elif modifier.type == "ARMATURE":
+                return False
+
+        if obj.parent is None:
+            break
+        obj = obj.parent
+    return True
+
+
 @gin.configurable
-def save_meshes(scene_seed, output_folder, frame_range, resample_idx=False):
+def save_meshes(
+    scene_seed,
+    output_folder,
+    frame_range,
+    resample_idx=False,
+    point_trajectory_src_frame=1,
+):
     if resample_idx is not None and resample_idx > 0:
         resample_scene(int_hash((scene_seed, resample_idx)))
 
     triangulate_meshes()
 
-    for obj in bpy.data.objects:
-        obj.hide_viewport = obj.hide_render
-
     for col in bpy.data.collections:
         col.hide_viewport = col.hide_render
 
     previous_frame_mesh_id_mapping = frozendict()
     current_frame_mesh_id_mapping = defaultdict(dict)
-    for frame_idx in range(int(frame_range[0]), int(frame_range[1] + 2)):
+
+    # save static meshes
+    for obj in bpy.data.objects:
+        obj.hide_viewport = not (not obj.hide_render and is_static(obj))
+    frame_idx = point_trajectory_src_frame
+    frame_info_folder = Path(output_folder) / f"frame_{frame_idx:04d}"
+    frame_info_folder.mkdir(parents=True, exist_ok=True)
+    logger.info("Working on static objects")
+    exporting.save_obj_and_instances(
+        frame_info_folder / "static_mesh",
+        previous_frame_mesh_id_mapping,
+        current_frame_mesh_id_mapping,
+    )
+    previous_frame_mesh_id_mapping = frozendict(current_frame_mesh_id_mapping)
+    current_frame_mesh_id_mapping.clear()
+
+    for obj in bpy.data.objects:
+        obj.hide_viewport = not (not obj.hide_render and not is_static(obj))
+
+    for frame_idx in set(
+        [point_trajectory_src_frame]
+        + list(range(int(frame_range[0]), int(frame_range[1] + 2)))
+    ):
         bpy.context.scene.frame_set(frame_idx)
         bpy.context.view_layer.update()
         frame_info_folder = Path(output_folder) / f"frame_{frame_idx:04d}"
@@ -143,6 +191,7 @@ def execute_tasks(
     reset_assets=True,
     dryrun=False,
     optimize_terrain_diskusage=False,
+    point_trajectory_src_frame=1,
 ):
     if input_folder != output_folder:
         if reset_assets:
@@ -283,9 +332,10 @@ def execute_tasks(
 
     if Task.MeshSave in task:
         save_meshes(
-            scene_seed,
-            output_folder=output_folder,
-            frame_range=frame_range,
+            scene_seed, 
+            output_folder=output_folder, 
+            frame_range=frame_range, 
+            point_trajectory_src_frame=point_trajectory_src_frame, 
         )
 
 

infinigen/core/util/exporting.py

@@ -199,11 +199,6 @@ def save_obj_and_instances(
     for atm_name in ["atmosphere", "atmosphere_fine", "KoleClouds"]:
         if atm_name in bpy.data.objects:
             bpy.data.objects.remove(bpy.data.objects[atm_name])
-    if "scatters" in bpy.data.collections:
-        for obj in bpy.data.collections["scatters"].objects:
-            if "instance_scatter" in obj.modifiers.keys():
-                obj.hide_viewport = False
-        bpy.data.collections["scatters"].hide_viewport = False
 
     json_data = []
     instance_mesh_data = get_all_instances()
@@ -318,6 +313,8 @@ def save_obj_and_instances(
         print(f"Saving to {filename}")
 
     for obj in bpy.data.objects:
+        if obj.hide_viewport:
+            continue
         if obj.type not in {"MESH", "CURVES", "CAMERA"}:
             object_name = obj.name
             if object_name not in object_names_mapping: