[Question] Help: Camera outputting same image to files despite moving in IsaacSim

[jbt-cmu]

Question

As my "rod" agent moves across the scene, so too does the affixed camera, meaning the saved image every 200 timesteps should change. However, it remains the same as the image provided upon reset. I'm not sure if this is a bug or something I've done, please help:

import argparse

from omni.isaac.lab.app import AppLauncher

# create argparser
parser = argparse.ArgumentParser(description="Tutorial on running IsaacSim via the AppLauncher.")
parser.add_argument("--size", type=float, default=1.0, help="Side-length of cuboid")
# SimulationApp arguments https://docs.omniverse.nvidia.com/py/isaacsim/source/extensions/omni.isaac.kit/docs/index.html?highlight=simulationapp#omni.isaac.kit.SimulationApp
parser.add_argument(
    "--width", type=int, default=1280, help="Width of the viewport and generated images. Defaults to 1280"
)
parser.add_argument(
    "--height", type=int, default=720, help="Height of the viewport and generated images. Defaults to 720"
)

# append AppLauncher cli args
AppLauncher.add_app_launcher_args(parser)
# parse the arguments
args_cli = parser.parse_args()
# launch omniverse app
app_launcher = AppLauncher(args_cli)
simulation_app = app_launcher.app

"""Rest everything follows."""

import omni.isaac.lab.sim as sim_utils
from omni.isaac.lab.sensors import Camera, CameraCfg
import omni.isaac.core.utils.prims as prim_utils
import gym
import cv2
import numpy as np
import torch
import os
import time
import random
from omni.isaac.lab.scene import InteractiveScene, InteractiveSceneCfg
from omni.isaac.lab.assets import AssetBaseCfg
from omni.isaac.lab.utils import configclass
from PIL import Image
from omni.isaac.lab.sim.spawners import RigidObjectSpawnerCfg
from omni.isaac.lab.sim import schemas

@configclass
class SimpleSceneCfg(InteractiveSceneCfg):
    """Design the scene with a rod and camera."""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        # Ground plane and lights (unchanged)
        self.ground = AssetBaseCfg(prim_path="/World/defaultGroundPlane", spawn=sim_utils.GroundPlaneCfg())
        self.dome_light = AssetBaseCfg(
            prim_path="/World/Light", spawn=sim_utils.DomeLightCfg(intensity=3000.0, color=(0.75, 0.75, 0.75))
        )

        # Rod (unchanged)
        self.rod = AssetBaseCfg(
            prim_path="{ENV_REGEX_NS}/Rod",
            spawn=sim_utils.CuboidCfg(
                size=[0.1,0.1, 3],
                visual_material=sim_utils.PreviewSurfaceCfg(
                    diffuse_color=(0.0, 0.1, 0.4)  # Set color to light blue
                ),
            ),
        )

        colors = {
            "red": (0.9, 0.0, 0.0),
            "green": (0.0, 0.9, 0.0),
            "blue": (0.0, 0.0, 0.9),
            "black": (0.0, 0.0, 0.0),
            "yellow": (0.9, 0.9, 0.0)
            }


        # Randomly select a position for x and y within the range [0, 10]
        x_position = random.randint(-5, 5)
        y_position = random.randint(-5, 5)
        print("position")
        print(x_position)
        print(y_position)

        # Spawn the block at the random position
        object_name = "object_1"
        setattr(self, object_name, AssetBaseCfg(
            prim_path=f"{{ENV_REGEX_NS}}/Object_1",
            spawn=sim_utils.CuboidCfg(size=[1, 1, 1], visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.9, 0.0, 0.0))),
            init_state=AssetBaseCfg.InitialStateCfg(pos=[x_position+0.5, y_position+0.5, 0.6])))

        # Camera (unchanged)
        self.camera = CameraCfg(
            prim_path="{ENV_REGEX_NS}/Rod/Camera",
            update_period=0.1,
            height=300,
            width=300,
            data_types=["rgb", "distance_to_image_plane"],
            spawn=sim_utils.PinholeCameraCfg(focal_length=24.0, focus_distance=100.0, horizontal_aperture=20.955, clipping_range=(0.1, 1.0e5)),
            offset=CameraCfg.OffsetCfg(pos=(0.2, 0.0, 1), rot= (0,0,-0.7071,0.7071), convention="ros"),
        )

        self.wall_1 = AssetBaseCfg(
            prim_path="{ENV_REGEX_NS}/Wall_1",
            spawn=sim_utils.CuboidCfg(
                size=[12,1, 2],
                visual_material=sim_utils.PreviewSurfaceCfg(
                    diffuse_color=(0.1, 0.1, 0.1)  # Set color to light blue
                ),
            ),
            init_state=AssetBaseCfg.InitialStateCfg(
                pos=([0.0, 5.5, 1])
            )
        )

        self.wall_2 = AssetBaseCfg(
            prim_path="{ENV_REGEX_NS}/Wall_2",
            spawn=sim_utils.CuboidCfg(
                size=[12,1, 2],
                visual_material=sim_utils.PreviewSurfaceCfg(
                    diffuse_color=(0.1, 0.1, 0.1)  # Set color to light blue
                ),
            ),
            init_state=AssetBaseCfg.InitialStateCfg(
                pos=([0.0, -5.5, 1])
            )
        )

        self.wall_3 = AssetBaseCfg(
            prim_path="{ENV_REGEX_NS}/Wall_3",
            spawn=sim_utils.CuboidCfg(
                size=[1,10, 2],
                visual_material=sim_utils.PreviewSurfaceCfg(
                    diffuse_color=(0.1, 0.1, 0.1)  # Set color to light blue
                ),
            ),
            init_state=AssetBaseCfg.InitialStateCfg(
                pos=([5.5, 0.0, 1])
            )
        )

        self.wall_4 = AssetBaseCfg(
            prim_path="{ENV_REGEX_NS}/Wall_4",
            spawn=sim_utils.CuboidCfg(
                size=[1,10, 2],
                visual_material=sim_utils.PreviewSurfaceCfg(
                    diffuse_color=(0.1, 0.1, 0.1)  # Set color to light blue
                ),
            ),
            init_state=AssetBaseCfg.InitialStateCfg(
                pos=([-5.5, 0.0, 1])
            )
        )

class CameraAgentEnv(gym.Env):
    def __init__(self):
        super(CameraAgentEnv, self).__init__()
        self.sim = self.create_simulation_context()
        self.scene = self.create_scene()
        self.image_counter = 0
        self.image_dir = "line_images"
        self.last_image_time = time.time()
        self.sim_time = 0
        
        # Define observation and action spaces
        self.observation_space = gym.spaces.Dict({
            "image": gym.spaces.Box(low=0, high=255, shape=(args_cli.height, args_cli.width, 3), dtype=np.uint8),
        })
        self.action_space = gym.spaces.Box(low=-1.0, high=1.0, shape=(3,), dtype=np.float32)

    def create_simulation_context(self):
        # Initialize the simulation context
        sim_cfg = sim_utils.SimulationCfg(dt=0.01)
        sim = sim_utils.SimulationContext(sim_cfg)
        # Set main camera view - zoomed out to see the entire scene
        sim.set_camera_view([5.0, 5.0, 5.0], [0.0, 0.0, 0.0])  # Adjust these values as needed for your scene
        return sim

    def create_scene(self):
        scene_cfg = SimpleSceneCfg(num_envs=1, env_spacing=2.0)
        scene = InteractiveScene(scene_cfg)
        return scene

    def reset(self):
        self.sim.reset()
        self.init_rod_position()
        return self.get_observations()
        
    def step(self, action):
        # Apply action to move the rod and camera
        self.apply_action(action)
        self.sim.step(render=True)
        observations = self.get_observations()
        self.sim_time += 1
        #(self.sim_time)

        # Save the image once per second
        if self.sim_time % 200 == 0 and "image" in observations:
           # input()
            image = observations["image"]
            image = image[0]
            print("imaging")
            

            #image = image[:,:,:]
            image = Image.fromarray(image)
            image_filename = os.path.join(self.image_dir, f"image_{self.image_counter:04d}.png")
            print(image_filename)
            image.save(image_filename)
            self.image_counter += 1

        reward = self.compute_reward()
        done = self.check_done()
        return observations, reward, done, {}

    def init_rod_position(self):
        # Initialize rod position to be slightly above the ground
        initial_position = torch.tensor([[0, 0, 0.5]], dtype=torch.float32, device=self.sim.device)
        self.scene["rod"].set_world_poses(positions=initial_position)

    def apply_action(self, action):
        # Apply the action to move the rod
        current_position = self.scene["rod"].get_world_poses()[0]
        new_position = current_position + torch.tensor(action, dtype=torch.float32, device=self.sim.device) * 0.01  # scale action for movement
        new_position = new_position.unsqueeze(0)  # Ensure correct shape
        self.scene["rod"].set_world_poses(positions=new_position)

    def get_observations(self):
        # Get the camera image
        image = self.scene['camera'].data.output["rgb"]
        image = image.cpu().numpy()  # Ensure it's a numpy array

        return {"image": image}


    def compute_reward(self):
        # Define reward calculation
        return 1.0
    
    def check_done(self):
        # Define termination conditions
        return False

def main():
    """Main function."""
    # Initialize the simulation context
    sim_cfg = sim_utils.SimulationCfg(dt=0.01)
    sim = sim_utils.SimulationContext(sim_cfg)
    # Set main camera
    sim.set_camera_view([5.0, 5.0, 5.0], [0.0, 0.0, 0.0])  # Zoomed out view

    # Design scene by adding assets to it
    env = CameraAgentEnv()
    env.reset()

    # Play the simulator
    sim.play()  # Start the simulation
    # Now we are ready!
    print("[INFO]: Setup complete...")

    # Simulate physics
    while simulation_app.is_running():
        # Generate random action
        action = env.action_space.sample()  # Random action

        action[0] = 1
        action[1] = 0
        action[2] = 0 
        for i in range(5000):
            observations, _, _, _ = env.step(action)

            image = observations["image"]

if __name__ == "__main__":
    # run the main function
    main()
    # close sim app
    simulation_app.close()

[pascal-roth]

Hi,
I assume that you placed your camera under the wrong prim. "{ENV_REGEX_NS}/Rod" will be the Xform under which the robot is spawned, but the actual articulation will be something like "{ENV_REGEX_NS}/Rod/base". For the camera to move with the robot, the camera would have to be placed under base.

As a reference, you can check the example here

[pascal-roth]

Did this fix your issue?