Neural Volume Rendering

NuRec (Neural Reconstruction) enables scene rendering in Omniverse using neural volumes derived from real-world images. These scenes, based on 3D Gaussian models, can be loaded into Isaac Sim as standard USD assets for visualization and simulation.

For more details on how NuRec works in Omniverse, including data preparation, rendering settings, and known limitations, see the NuRec documentation. To generate compatible scenes, you can use the open-source project 3DGruT which provides tools for training 3D Gaussian models from image collections and exporting them in a USDZ-based format suitable for use in Omniverse applications.

Example

The following example demonstrates how to load a NuRec scene into Isaac Sim and run a simulation. The snippet iterates over the provided examples and starts by loading the provided stage, it then loads the carter navigation asset and sets the start location. It then checks if a collision ground plane needs to be created at the spawn location, and if so, creates a plane prim with a collision API applied. It then sets the carter navigation target prim location and runs the simulation for the given number of steps. During the simulation the wheeled robot will navigate towards the target location.

The example script can be run directly from the Script Editor or as a Standalone Application.

Prerequisites

• Download the NVIDIA NuRec Dataset from Hugging Face.

• Update the USER_PATH variable in the script: USER_PATH = "/home/user/PhysicalAI-Robotics-NuRec"

Script Editor

Script Editor

import asyncio
import os

import omni.kit.commands
import omni.kit.app
import omni.usd
import omni.timeline
from isaacsim.storage.native import get_assets_root_path_async
from isaacsim.core.utils.stage import add_reference_to_stage
from pxr import PhysxSchema, UsdGeom, UsdPhysics

# User path of the HF NuRec dataset
USER_PATH = "/home/user/PhysicalAI-Robotics-NuRec"

# Paths for loading and placing the Nova Carter navigation asset and its target.
NOVA_CARTER_NAV_URL = "/Isaac/Samples/Replicator/OmniGraph/nova_carter_nav_only.usd"
NOVA_CARTER_NAV_USD_PATH = "/World/NovaCarterNav"
NOVA_CARTER_NAV_TARGET_PATH = f"{NOVA_CARTER_NAV_USD_PATH}/targetXform"
# Scenarios for testing navigation in the environments
EXAMPLE_CONFIGS = [
   {
      "name": "Voyager Cafe",
      "stage_url": f"{USER_PATH}/nova_carter-cafe/stage.usdz",
      "nav_start_loc": (0, 0, 0),
      "nav_relative_target_loc": (-3, -1.5, 0),
      "create_collision_ground_plane": False,
      "num_simulation_steps": 500,
   },
   {
      "name": "Galileo Lab",
      "stage_url": f"{USER_PATH}/nova_carter-galileo/stage.usdz",
      "nav_start_loc": (3.5, 2.5, 0),
      "nav_relative_target_loc": (4, 0, 0),
      "create_collision_ground_plane": False,
      "num_simulation_steps": 500,
   },
   {
      "name": "Wormhole",
      "stage_url": f"{USER_PATH}/nova_carter-wormhole/stage.usdz",
      "nav_start_loc": (0, 0, 0),
      "nav_relative_target_loc": (5, 0, 0),
      "create_collision_ground_plane": False,
      "num_simulation_steps": 500,
   },
   {
      "name": "ZH Lounge",
      "stage_url": f"{USER_PATH}/zh_lounge/usd/zh_lounge.usda",
      "nav_start_loc": (-1.5, -3, -1.6),
      "nav_relative_target_loc": (-0.5, 5, -1.6),
      "create_collision_ground_plane": True,
      "num_simulation_steps": 500,
   },
]


async def run_example_async(example_config):
   example_name = example_config.get("name")
   print(f"Running example: '{example_name}'")

   # Open the stage
   stage_url = example_config.get("stage_url")
   if not stage_url:
      print(f"Stage URL not provided, exiting")
      return
   if not os.path.exists(stage_url):
      print(f"Stage URL does not exist: '{stage_url}', exiting")
      return

   print(f"Opening stage: '{stage_url}'")
   await omni.usd.get_context().open_stage_async(stage_url)
   stage = omni.usd.get_context().get_stage()

   # Make sure the physics scene is set to synchronous for the navigation to work
   for prim in stage.Traverse():
      if prim.IsA(UsdPhysics.Scene):
            physx_scene = PhysxSchema.PhysxSceneAPI.Apply(prim)
            physx_scene.GetUpdateTypeAttr().Set("Synchronous")
            break

   # Load the carter navigation asset
   assets_root_path = await get_assets_root_path_async()
   carter_nav_path = assets_root_path + NOVA_CARTER_NAV_URL
   print(f"Loading carter nova asset: '{carter_nav_path}'")
   carter_nav_prim = add_reference_to_stage(usd_path=carter_nav_path, prim_path=NOVA_CARTER_NAV_USD_PATH)

   # Set the carter navigation start location
   nav_start_loc = example_config.get("nav_start_loc")
   if not nav_start_loc:
      print(f"Navigation start location not provided, exiting")
      return
   print(f"Setting carter navigation start location to: {nav_start_loc}")
   if not carter_nav_prim.GetAttribute("xformOp:translate"):
      UsdGeom.Xformable(carter_nav_prim).AddTranslateOp()
   carter_nav_prim.GetAttribute("xformOp:translate").Set(nav_start_loc)

   # Check if a collision ground plane needs to be created at the spawn location
   if example_config.get("create_collision_ground_plane"):
      plane_path = "/World/CollisionPlane"
      print(f"Creating collision ground plane {plane_path} at {nav_start_loc}")
      omni.kit.commands.execute("CreateMeshPrimWithDefaultXform", prim_path=plane_path, prim_type="Plane")
      plane_prim = stage.GetPrimAtPath(plane_path)
      plane_prim.GetAttribute("xformOp:scale").Set((10, 10, 1))
      plane_prim.GetAttribute("xformOp:translate").Set(nav_start_loc)
      if not plane_prim.HasAPI(UsdPhysics.CollisionAPI):
            collision_api = UsdPhysics.CollisionAPI.Apply(plane_prim)
      else:
            collision_api = UsdPhysics.CollisionAPI(plane_prim)
      collision_api.CreateCollisionEnabledAttr(True)
      plane_prim.GetAttribute("visibility").Set("invisible")

   # Set the carter navigation target prim location
   nav_relative_target_loc = example_config.get("nav_relative_target_loc")
   if not nav_relative_target_loc:
      print(f"Navigation relative target location not provided, exiting")
      return
   print(f"Setting carter navigation target location to: {nav_relative_target_loc}")
   carter_navigation_target_prim = stage.GetPrimAtPath(NOVA_CARTER_NAV_TARGET_PATH)
   if not carter_navigation_target_prim.IsValid():
      print(f"Carter navigation target prim not found at path: '{NOVA_CARTER_NAV_TARGET_PATH}', exiting")
      return
   if not carter_navigation_target_prim.GetAttribute("xformOp:translate"):
      UsdGeom.Xformable(carter_navigation_target_prim).AddTranslateOp()
   carter_navigation_target_prim.GetAttribute("xformOp:translate").Set(nav_relative_target_loc)

   # Run the simulation for the given number of steps
   num_simulation_steps = example_config.get("num_simulation_steps")
   if not num_simulation_steps:
      print(f"Number of simulation steps not provided, exiting")
      return
   print(f"Running {num_simulation_steps} simulation steps")
   timeline = omni.timeline.get_timeline_interface()
   timeline.play()
   for i in range(num_simulation_steps):
      if i % 10 == 0:
            print(f"Step {i}, time: {timeline.get_current_time():.4f}")
      await omni.kit.app.get_app().next_update_async()

   print(f"Simulation complete, pausing timeline")
   timeline.pause()


async def run_examples_async():
   for example_config in EXAMPLE_CONFIGS:
      await run_example_async(example_config)


asyncio.ensure_future(run_examples_async())

Standalone Application

Standalone Application

import os

from isaacsim import SimulationApp

simulation_app = SimulationApp(launch_config={"headless": False})

import omni.kit.commands
import omni.kit.app
import omni.usd
import omni.timeline
from isaacsim.storage.native import get_assets_root_path
from isaacsim.core.utils.stage import add_reference_to_stage
from pxr import PhysxSchema, UsdGeom, UsdPhysics

# User path of the HF NuRec dataset
USER_PATH = "/home/user/PhysicalAI-Robotics-NuRec"

# Paths for loading and placing the Nova Carter navigation asset and its target.
NOVA_CARTER_NAV_URL = "/Isaac/Samples/Replicator/OmniGraph/nova_carter_nav_only.usd"
NOVA_CARTER_NAV_USD_PATH = "/World/NovaCarterNav"
NOVA_CARTER_NAV_TARGET_PATH = f"{NOVA_CARTER_NAV_USD_PATH}/targetXform"
# Scenarios for testing navigation in the environments
EXAMPLE_CONFIGS = [
   {
      "name": "Voyager Cafe",
      "stage_url": f"{USER_PATH}/nova_carter-cafe/stage.usdz",
      "nav_start_loc": (0, 0, 0),
      "nav_relative_target_loc": (-3, -1.5, 0),
      "create_collision_ground_plane": False,
      "num_simulation_steps": 500,
   },
   {
      "name": "Galileo Lab",
      "stage_url": f"{USER_PATH}/nova_carter-galileo/stage.usdz",
      "nav_start_loc": (3.5, 2.5, 0),
      "nav_relative_target_loc": (4, 0, 0),
      "create_collision_ground_plane": False,
      "num_simulation_steps": 500,
   },
   {
      "name": "Wormhole",
      "stage_url": f"{USER_PATH}/nova_carter-wormhole/stage.usdz",
      "nav_start_loc": (0, 0, 0),
      "nav_relative_target_loc": (5, 0, 0),
      "create_collision_ground_plane": False,
      "num_simulation_steps": 500,
   },
   {
      "name": "ZH Lounge",
      "stage_url": f"{USER_PATH}/zh_lounge/usd/zh_lounge.usda",
      "nav_start_loc": (-1.5, -3, -1.6),
      "nav_relative_target_loc": (-0.5, 5, -1.6),
      "create_collision_ground_plane": True,
      "num_simulation_steps": 500,
   },
]


def run_example(example_config):
   example_name = example_config.get("name")
   print(f"Running example: '{example_name}'")

   # Open the stage
   stage_url = example_config.get("stage_url")
   if not stage_url:
      print(f"Stage URL not provided, exiting")
      return
   if not os.path.exists(stage_url):
      print(f"Stage URL does not exist: '{stage_url}', exiting")
      return

   print(f"Opening stage: '{stage_url}'")
   omni.usd.get_context().open_stage(stage_url)
   stage = omni.usd.get_context().get_stage()

   # Make sure the physics scene is set to synchronous for the navigation to work
   for prim in stage.Traverse():
      if prim.IsA(UsdPhysics.Scene):
            physx_scene = PhysxSchema.PhysxSceneAPI.Apply(prim)
            physx_scene.GetUpdateTypeAttr().Set("Synchronous")
            break

   # Load the carter navigation asset
   assets_root_path = get_assets_root_path()
   carter_nav_path = assets_root_path + NOVA_CARTER_NAV_URL
   print(f"Loading carter nova asset: '{carter_nav_path}'")
   carter_nav_prim = add_reference_to_stage(usd_path=carter_nav_path, prim_path=NOVA_CARTER_NAV_USD_PATH)

   # Set the carter navigation start location
   nav_start_loc = example_config.get("nav_start_loc")
   if not nav_start_loc:
      print(f"Navigation start location not provided, exiting")
      return
   print(f"Setting carter navigation start location to: {nav_start_loc}")
   if not carter_nav_prim.GetAttribute("xformOp:translate"):
      UsdGeom.Xformable(carter_nav_prim).AddTranslateOp()
   carter_nav_prim.GetAttribute("xformOp:translate").Set(nav_start_loc)

   # Check if a collision ground plane needs to be created at the spawn location
   if example_config.get("create_collision_ground_plane"):
      plane_path = "/World/CollisionPlane"
      print(f"Creating collision ground plane {plane_path} at {nav_start_loc}")
      omni.kit.commands.execute("CreateMeshPrimWithDefaultXform", prim_path=plane_path, prim_type="Plane")
      plane_prim = stage.GetPrimAtPath(plane_path)
      plane_prim.GetAttribute("xformOp:scale").Set((10, 10, 1))
      plane_prim.GetAttribute("xformOp:translate").Set(nav_start_loc)
      if not plane_prim.HasAPI(UsdPhysics.CollisionAPI):
            collision_api = UsdPhysics.CollisionAPI.Apply(plane_prim)
      else:
            collision_api = UsdPhysics.CollisionAPI(plane_prim)
      collision_api.CreateCollisionEnabledAttr(True)
      plane_prim.GetAttribute("visibility").Set("invisible")

   # Set the carter navigation target prim location
   nav_relative_target_loc = example_config.get("nav_relative_target_loc")
   if not nav_relative_target_loc:
      print(f"Navigation relative target location not provided, exiting")
      return
   print(f"Setting carter navigation target location to: {nav_relative_target_loc}")
   carter_navigation_target_prim = stage.GetPrimAtPath(NOVA_CARTER_NAV_TARGET_PATH)
   if not carter_navigation_target_prim.IsValid():
      print(f"Carter navigation target prim not found at path: '{NOVA_CARTER_NAV_TARGET_PATH}', exiting")
      return
   if not carter_navigation_target_prim.GetAttribute("xformOp:translate"):
      UsdGeom.Xformable(carter_navigation_target_prim).AddTranslateOp()
   carter_navigation_target_prim.GetAttribute("xformOp:translate").Set(nav_relative_target_loc)

   # Run the simulation for the given number of steps
   num_simulation_steps = example_config.get("num_simulation_steps")
   if not num_simulation_steps:
      print(f"Number of simulation steps not provided, exiting")
      return
   print(f"Running {num_simulation_steps} simulation steps")
   timeline = omni.timeline.get_timeline_interface()
   timeline.play()
   for i in range(num_simulation_steps):
      if i % 10 == 0:
            print(f"Step {i}, time: {timeline.get_current_time():.4f}")
      simulation_app.update()

   print(f"Simulation complete, pausing timeline")
   timeline.pause()


def run_examples():
   for example_config in EXAMPLE_CONFIGS:
      run_example(example_config)


run_examples()

simulation_app.close()