Hi, Sapien simulator give wrong mask after two sequential simulations

[Tianyi20]

Hi, appreciate your great work of sapien. And I hope to report a problem that seems like an issue.

I appreciate a lot if you can help!

Here is my code that I sequencly create scene and add corresponding entities inside the scene. The virtual environment is a list that contains all obj file and collision obj file. In each scene, End effector push the objects and record the final image and calculate the loss compared with GT image. In the output of each scene, the generated RGB image, point cloud, depth, video is right sequencely.

However, the mask output from camera is wrong after two sequence scene simulations. I observe the object entites ID is increasing based on the last scene, but the mask output ID is still within 0 1 2 3.

ic| f"{env_index} envrionment entities ID are": '0 envrionment entities ID are'    objects_ID: [2, 3, 4]
ic| f"{env_index} envrionment entities ID are": '1 envrionment entities ID are'  objects_ID: [9, 10, 11, 12] 
ic| f"{env_index} envrionment entities ID are": '2 envrionment entities ID are'    objects_ID: [17, 18, 19, 20]

Here is my camera getting picture function. I rewrite from tutorial camera doc.

def getCameraImage_wrap(camera, if_write = False, save_path = "", filename ="", if_visualize_pcd = False):
    camera.take_picture()  # submit rendering jobs to the GPU
    seg_labels = camera.get_picture("Segmentation")  # [H, W, 4]
    label0_image = seg_labels[..., 1]  # mesh-level
    mask = label0_image

And I have also test the seg_labels[..., 0] for both mesh-level and actor level. However, the output is still wrong that mask output ID is still within 0 1 2 3.

virtual_environments = []

for env_index, environment in enumerate(virtual_environments):
    sim_save_path = f"{save_dir}/env{env_index}/simulation"
    if not os.path.exists(sim_save_path):
        os.makedirs(sim_save_path, exist_ok= True)

    """Set scene default configuration"""
    scene_config = sapien.physx.PhysxSceneConfig()
    shape_config = sapien.physx.PhysxShapeConfig()
    #print(shape_config.contact_offset)
    shape_config.contact_offset = 0.0002
    shape_config.rest_offset = 0.0001

    ic(shape_config.contact_offset)
    ic(shape_config.rest_offset)
    # The default physical properties can be modified through sapien.SceneConfig
    print(scene_config.gravity)
    scene_config.gravity = np.array([0.0, 0.0, -9.8])
    sapien.physx.set_scene_config(scene_config)
    sapien.physx.set_shape_config(shape_config)

    # SAPIEN's default physical material for PhysX can be modified at any time
    # It is not bound to a scene
    default_material = sapien.physx.get_default_material()
    print(default_material.static_friction)
    print(default_material.dynamic_friction)
    print(default_material.restitution)
    sapien.physx.set_default_material(
        static_friction=0.3, dynamic_friction=0.3, restitution=0.1)

    # by default, SAPIEN scene consists of PhysxSystem and RenderSystem
    # The PhysxSystem can take a PhysxSceneConfig to custom its behavior
    scene = sapien.Scene([
        sapien.physx.PhysxCpuSystem(),
        sapien.render.RenderSystem(),
    ])

    physical_material: sapien.physx.PhysxMaterial = sapien.physx.PhysxMaterial(
        static_friction= 0.3,
        dynamic_friction=0.3,
        restitution= 0.1,
    )

    scene.set_timestep(1 / 240.0)  # Set the simulation frequency
    ground = scene.add_ground(altitude=0)  # Add a ground
    ground_id = ground.get_global_id()
    ic(ground_id)
    #exit()
    objects_ID = []
    entities = []
    for object_index, object in enumerate(environment):
        body_entity = create_body(scene=scene,
                                visual_mesh_filename= object["convex_decomposed_filename"],
                                collision_mesh_filename= object["convex_decomposed_filename"],
                                pose= None,
                                density= object["density"],
                                physical_material= physical_material,
                                name= f"env{env_index}_body_{object_index}",
                                linear_damping= 0.2,
                                angular_damping= 0.0)
        objects_ID.append(body_entity.get_global_id())
        entities.append(body_entity)

    ic(f"{env_index} envrionment entities ID are",objects_ID)
    # Add some lights so that you can observe the scene
    scene.set_ambient_light([0.5, 0.5, 0.5])
    scene.add_directional_light([0, 1, -1], [0.5, 0.5, 0.5])

    times, milstones = load_ee_pose(file_name=trajectory_dir, discretize= 1/240, if_discrete= True)

    EE = create_EE_as_sphere(scene=scene, pose= sapien.Pose(p=milstones[0]), radius= 0.02, color = [0.5, 0.5, 0.5], is_kinematic=False, density= 5000, physical_material= physical_material)
    entities.append(EE)

    camera = setup_camera(scene=scene, configs=configs)

    rgba_img, points_world, points_color, pcd, depth_img, depth, mask = getCameraImage_wrap(camera=camera, if_write=True, save_path= sim_save_path, filename= "Initial_scene", if_visualize_pcd= True )
    print(f"{env_index} generated mask is:\n")
    #print(np.array2string(mask, threshold=np.inf))
    loss_initial = loss_sep_obj(pre_pcd=points_world, gt_pcd= origin_objects_pcd_before, mask= mask.flatten(), objects_ID= objects_ID)
    ic(loss_initial)
    counter = 0
    frames = []
        
    #viewer = scene.create_viewer()  # Create a viewer (window)

    while counter < len(times):
        scene.step()
        scene.update_render()
        EE.set_pose(sapien.Pose(p = milstones[counter]))
        if if_give_video and counter % 100 == 0:
                rgba_img, points_world, points_color, pcd, depth_img, depth, mask = getCameraImage_wrap(camera=camera, if_write=False, if_visualize_pcd= False )
                frames.append(rgba_img[:, :, [2, 1, 0]])
        counter += 1
        #viewer.render()
        #time.sleep(0.01)
    rgba_img, points_world, points_color, pcd, depth_img, depth, mask = getCameraImage_wrap(camera=camera, if_write=True, save_path= sim_save_path, filename= "Interacted_scene", if_visualize_pcd= True )
    if if_give_video:
        generate_video_from_rgb(frames, sim_save_path, "sim_video", fps= 2.4)
    

[fbxiang]

The id you want is probably per_scene_id instead of global id. Global id is monotonically increasing when creating entity so you can tell apart any 2 entities. The per-scene id is managed by each scene and increases as entities are added to scene. The renderer renders per-scene id so you can get the same id for different scenes.

[Tianyi20]

got it. Thank you very much for your help!