[Newton] Adding visualizer support for Newton

scripts/environments/random_agent.py

@@ -18,7 +18,6 @@
 )
 parser.add_argument("--num_envs", type=int, default=None, help="Number of environments to simulate.")
 parser.add_argument("--task", type=str, default=None, help="Name of the task.")
-parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.")
 # append AppLauncher cli args
 AppLauncher.add_app_launcher_args(parser)
 # parse the arguments
@@ -52,8 +51,8 @@ def main():
         device=args_cli.device,
         num_envs=args_cli.num_envs,
         use_fabric=not args_cli.disable_fabric,
-        newton_visualizer=args_cli.newton_visualizer,
     )
+
     # create environment
     env = gym.make(args_cli.task, cfg=env_cfg)
 

scripts/environments/zero_agent.py

@@ -18,7 +18,6 @@
 )
 parser.add_argument("--num_envs", type=int, default=None, help="Number of environments to simulate.")
 parser.add_argument("--task", type=str, default=None, help="Name of the task.")
-parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.")
 # append AppLauncher cli args
 AppLauncher.add_app_launcher_args(parser)
 # parse the arguments
@@ -52,8 +51,8 @@ def main():
         device=args_cli.device,
         num_envs=args_cli.num_envs,
         use_fabric=not args_cli.disable_fabric,
-        newton_visualizer=args_cli.newton_visualizer,
     )
+
     # create environment
     env = gym.make(args_cli.task, cfg=env_cfg)
 

scripts/reinforcement_learning/rl_games/play.py

@@ -32,7 +32,6 @@
     help="When no checkpoint provided, use the last saved model. Otherwise use the best saved model.",
 )
 parser.add_argument("--real-time", action="store_true", default=False, help="Run in real-time, if possible.")
-parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.")
 # append AppLauncher cli args
 AppLauncher.add_app_launcher_args(parser)
 # parse the arguments
@@ -80,12 +79,9 @@ def main():
     task_name = args_cli.task.split(":")[-1]
     # parse env configuration
     env_cfg = parse_env_cfg(
-        args_cli.task,
-        device=args_cli.device,
-        num_envs=args_cli.num_envs,
-        use_fabric=not args_cli.disable_fabric,
-        newton_visualizer=args_cli.newton_visualizer,
+        args_cli.task, device=args_cli.device, num_envs=args_cli.num_envs, use_fabric=not args_cli.disable_fabric
     )
+
     agent_cfg = load_cfg_from_registry(args_cli.task, "rl_games_cfg_entry_point")
 
     # specify directory for logging experiments

scripts/reinforcement_learning/rl_games/train.py

@@ -38,7 +38,6 @@
     const=True,
     help="if toggled, this experiment will be tracked with Weights and Biases",
 )
-parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.")
 # append AppLauncher cli args
 AppLauncher.add_app_launcher_args(parser)
 # parse the arguments
@@ -90,7 +89,6 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: dict):
     # override configurations with non-hydra CLI arguments
     env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs
     env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device
-    env_cfg.sim.enable_newton_rendering = args_cli.newton_visualizer
 
     # randomly sample a seed if seed = -1
     if args_cli.seed == -1:

scripts/reinforcement_learning/rsl_rl/play.py

@@ -34,7 +34,6 @@
     help="Use the pre-trained checkpoint from Nucleus.",
 )
 parser.add_argument("--real-time", action="store_true", default=False, help="Run in real-time, if possible.")
-parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.")
 # append RSL-RL cli arguments
 cli_args.add_rsl_rl_args(parser)
 # append AppLauncher cli args
@@ -96,7 +95,6 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: RslRlBaseRun
     # note: certain randomizations occur in the environment initialization so we set the seed here
     env_cfg.seed = agent_cfg.seed
     env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device
-    env_cfg.sim.enable_newton_rendering = args_cli.newton_visualizer
 
     # specify directory for logging experiments
     log_root_path = os.path.join("logs", "rsl_rl", agent_cfg.experiment_name)
@@ -117,19 +115,6 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: RslRlBaseRun
     # set the log directory for the environment (works for all environment types)
     env_cfg.log_dir = log_dir
 
-    # Set play mode for Newton viewer if using Newton visualizer
-    if args_cli.newton_visualizer:
-        # Set visualizer to play mode in Newton config
-        if hasattr(env_cfg.sim, "newton_cfg"):
-            env_cfg.sim.newton_cfg.visualizer_train_mode = False
-        else:
-            # Create newton_cfg if it doesn't exist
-            from isaaclab.sim._impl.newton_manager_cfg import NewtonCfg
-
-            newton_cfg = NewtonCfg()
-            newton_cfg.visualizer_train_mode = False
-            env_cfg.sim.newton_cfg = newton_cfg
-
     # create isaac environment
     env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None)
 

scripts/reinforcement_learning/rsl_rl/train.py

@@ -32,7 +32,6 @@
     "--distributed", action="store_true", default=False, help="Run training with multiple GPUs or nodes."
 )
 parser.add_argument("--export_io_descriptors", action="store_true", default=False, help="Export IO descriptors.")
-parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.")
 # append RSL-RL cli arguments
 cli_args.add_rsl_rl_args(parser)
 # append AppLauncher cli args
@@ -122,7 +121,6 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: RslRlBaseRun
     # note: certain randomizations occur in the environment initialization so we set the seed here
     env_cfg.seed = agent_cfg.seed
     env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device
-    env_cfg.sim.enable_newton_rendering = args_cli.newton_visualizer
 
     # multi-gpu training configuration
     if args_cli.distributed:

scripts/reinforcement_learning/sb3/play.py

@@ -39,7 +39,6 @@
     default=False,
     help="Use a slower SB3 wrapper but keep all the extra training info.",
 )
-parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.")
 # append AppLauncher cli args
 AppLauncher.add_app_launcher_args(parser)
 # parse the arguments
@@ -83,11 +82,7 @@ def main():
     """Play with stable-baselines agent."""
     # parse configuration
     env_cfg = parse_env_cfg(
-        args_cli.task,
-        device=args_cli.device,
-        num_envs=args_cli.num_envs,
-        use_fabric=not args_cli.disable_fabric,
-        newton_visualizer=args_cli.newton_visualizer,
+        args_cli.task, device=args_cli.device, num_envs=args_cli.num_envs, use_fabric=not args_cli.disable_fabric
     )
 
     task_name = args_cli.task.split(":")[-1]

scripts/reinforcement_learning/sb3/train.py

@@ -32,7 +32,6 @@
     default=False,
     help="Use a slower SB3 wrapper but keep all the extra training info.",
 )
-parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.")
 # append AppLauncher cli args
 AppLauncher.add_app_launcher_args(parser)
 # parse the arguments
@@ -113,7 +112,6 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: dict):
     # note: certain randomizations occur in the environment initialization so we set the seed here
     env_cfg.seed = agent_cfg["seed"]
     env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device
-    env_cfg.sim.enable_newton_rendering = args_cli.newton_visualizer
 
     # directory for logging into
     run_info = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")

scripts/reinforcement_learning/skrl/play.py

@@ -46,7 +46,6 @@
     help="The RL algorithm used for training the skrl agent.",
 )
 parser.add_argument("--real-time", action="store_true", default=False, help="Run in real-time, if possible.")
-parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.")
 
 # append AppLauncher cli args
 AppLauncher.add_app_launcher_args(parser)
@@ -112,11 +111,7 @@ def main():
 
     # parse configuration
     env_cfg = parse_env_cfg(
-        args_cli.task,
-        device=args_cli.device,
-        num_envs=args_cli.num_envs,
-        use_fabric=not args_cli.disable_fabric,
-        newton_visualizer=args_cli.newton_visualizer,
+        args_cli.task, device=args_cli.device, num_envs=args_cli.num_envs, use_fabric=not args_cli.disable_fabric
     )
     try:
         experiment_cfg = load_cfg_from_registry(task_name, f"skrl_{algorithm}_cfg_entry_point")

scripts/reinforcement_learning/skrl/train.py

@@ -44,7 +44,6 @@
     choices=["AMP", "PPO", "IPPO", "MAPPO"],
     help="The RL algorithm used for training the skrl agent.",
 )
-parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.")
 
 # append AppLauncher cli args
 AppLauncher.add_app_launcher_args(parser)
@@ -113,7 +112,6 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, agent_cfg: dict):
     # override configurations with non-hydra CLI arguments
     env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs
     env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device
-    env_cfg.sim.enable_newton_rendering = args_cli.newton_visualizer
 
     # multi-gpu training config
     if args_cli.distributed:

scripts/sim2sim_transfer/rsl_rl_transfer.py

@@ -29,7 +29,6 @@
     help="Use the pre-trained checkpoint from Nucleus.",
 )
 parser.add_argument("--real-time", action="store_true", default=False, help="Run in real-time, if possible.")
-parser.add_argument("--newton_visualizer", action="store_true", default=False, help="Enable Newton rendering.")
 # Joint ordering arguments
 parser.add_argument(
     "--policy_transfer_file",
@@ -147,8 +146,8 @@ def main():
         device=args_cli.device,
         num_envs=args_cli.num_envs,
         use_fabric=not args_cli.disable_fabric,
-        newton_visualizer=args_cli.newton_visualizer,
     )
+
     agent_cfg: RslRlOnPolicyRunnerCfg = cli_args.parse_rsl_rl_cfg(task_name, args_cli)
 
     # specify directory for logging experiments

source/isaaclab/isaaclab/app/app_launcher.py

@@ -118,6 +118,9 @@ def __init__(self, launcher_args: argparse.Namespace | dict | None = None, **kwa
         self._livestream: Literal[0, 1, 2]  # 0: Disabled, 1: WebRTC public, 2: WebRTC private
         self._offscreen_render: bool  # 0: Disabled, 1: Enabled
         self._sim_experience_file: str  # Experience file to load
+        self._visualizer: (
+            list[str] | None
+        )  # Visualizer backends to use: None or list of ["rerun", "newton", "omniverse"]
 
         # Exposed to train scripts
         self.device_id: int  # device ID for GPU simulation (defaults to 0)
@@ -177,6 +180,16 @@ def app(self) -> SimulationApp:
         else:
             raise RuntimeError("The `AppLauncher.app` member cannot be retrieved until the class is initialized.")
 
+    @property
+    def visualizer(self) -> list[str] | None:
+        """The visualizer backend(s) to use.
+
+        Returns:
+            List of visualizer backend names (e.g., ["rerun", "newton"]) or None if no visualizers specified.
+            Empty list means no visualizers should be initialized.
+        """
+        return self._visualizer
+
     """
     Operations.
     """
@@ -229,6 +242,21 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None:
           Arguments should be combined into a single string separated by space.
           Example usage: --kit_args "--ext-folder=/path/to/ext1 --ext-folder=/path/to/ext2"
 
+        * ``visualizer`` (list[str]): The visualizer backend(s) to use for the simulation.
+          Valid options are:
+
+          - ``rerun``: Use Rerun visualizer.
+          - ``newton``: Use Newton visualizer.
+          - ``omniverse``: Use Omniverse visualizer.
+          - Multiple visualizers can be specified: ``--visualizer rerun newton``
+          - If not specified (default), NO visualizers will be initialized and headless mode is auto-enabled.
+
+          Note: If visualizer configs are not defined in the simulation config, default configs will be
+          automatically created with all default parameters.
+          If --headless is specified, it takes precedence and NO visualizers will be initialized.
+          When omniverse visualizer is specified, the app will launch in non-headless mode automatically.
+          When only non-GUI visualizers (rerun, newton) are specified, headless mode is auto-enabled.
+
         Args:
             parser: An argument parser instance to be extended with the AppLauncher specific options.
         """
@@ -365,6 +393,19 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None:
                 " exceeded, then the animation is not recorded."
             ),
         )
+        arg_group.add_argument(
+            "--visualizer",
+            type=str,
+            nargs="*",
+            default=None,
+            help=(
+                "The visualizer backend(s) to use for the simulation."
+                ' Can be one or more of: "rerun", "newton", "omniverse".'
+                " Multiple visualizers can be specified: --visualizer rerun newton."
+                " If not specified (default), NO visualizers will be created and headless mode is auto-enabled."
+                " If no visualizer configs are defined in sim config, default configs will be created automatically."
+            ),
+        )
         # special flag for backwards compatibility
 
         # Corresponding to the beginning of the function,
@@ -385,6 +426,7 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None:
         "device": ([str], "cuda:0"),
         "experience": ([str], ""),
         "rendering_mode": ([str], "balanced"),
+        "visualizer": ([list, type(None)], None),
     }
     """A dictionary of arguments added manually by the :meth:`AppLauncher.add_app_launcher_args` method.
 
@@ -482,6 +524,7 @@ def _config_resolution(self, launcher_args: dict):
         # Handle core settings
         livestream_arg, livestream_env = self._resolve_livestream_settings(launcher_args)
         self._resolve_headless_settings(launcher_args, livestream_arg, livestream_env)
+        self._resolve_visualizer_settings(launcher_args)
         self._resolve_camera_settings(launcher_args)
         self._resolve_xr_settings(launcher_args)
         self._resolve_viewport_settings(launcher_args)
@@ -573,9 +616,16 @@ def _resolve_headless_settings(self, launcher_args: dict, livestream_arg: int, l
             raise ValueError(
                 f"Invalid value for environment variable `HEADLESS`: {headless_env} . Expected: {headless_valid_vals}."
             )
+
+        # Check if visualizers are requested and if omniverse visualizer is among them
+        visualizers = launcher_args.get("visualizer", AppLauncher._APPLAUNCHER_CFG_INFO["visualizer"][1])
+        visualizers_requested = visualizers is not None and len(visualizers) > 0
+        omniverse_visualizer_requested = visualizers_requested and "omniverse" in visualizers
+
         # We allow headless kwarg to supersede HEADLESS envvar if headless_arg does not have the default value
         # Note: Headless is always true when livestreaming
         if headless_arg is True:
+            # User explicitly requested headless mode
             self._headless = headless_arg
         elif self._livestream in {1, 2}:
             # we are always headless on the host machine
@@ -591,12 +641,56 @@ def _resolve_headless_settings(self, launcher_args: dict, livestream_arg: int, l
                     f"[INFO][AppLauncher]: Environment variable `LIVESTREAM={self._livestream}` has implicitly"
                     f" overridden the environment variable `HEADLESS={headless_env}` to True."
                 )
+        elif omniverse_visualizer_requested and headless_env == 0:
+            # Omniverse visualizer requires non-headless mode (needs Isaac Sim viewport)
+            self._headless = False
+            print(
+                "[INFO][AppLauncher]: Omniverse visualizer requested via --visualizer flag. "
+                "Launching in non-headless mode to enable viewport visualization."
+            )
+        elif visualizers_requested and not omniverse_visualizer_requested and headless_env == 0:
+            # Newton and Rerun don't need Isaac Sim viewport (they create their own windows or are web-based)
+            self._headless = True
+            print(
+                f"[INFO][AppLauncher]: Visualizer(s) {visualizers} requested. "
+                "Enabling headless mode (Isaac Sim viewport disabled)."
+            )
+        elif not visualizers_requested and headless_env == 0:
+            # No visualizers requested and headless not explicitly set -> enable headless mode
+            self._headless = True
+            print(
+                "[INFO][AppLauncher]: No visualizers specified via --visualizer flag. "
+                "Automatically enabling headless mode. Use --visualizer <type> to enable GUI."
+            )
         else:
             # Headless needs to be a bool to be ingested by SimulationApp
             self._headless = bool(headless_env)
         # Headless needs to be passed to the SimulationApp so we keep it here
         launcher_args["headless"] = self._headless
 
+    def _resolve_visualizer_settings(self, launcher_args: dict):
+        """Resolve visualizer related settings."""
+        # Get visualizer setting from launcher_args
+        visualizers = launcher_args.get("visualizer", AppLauncher._APPLAUNCHER_CFG_INFO["visualizer"][1])
+
+        # Validate visualizer names
+        valid_visualizers = ["rerun", "newton", "omniverse"]
+        if visualizers is not None and len(visualizers) > 0:
+            invalid = [v for v in visualizers if v not in valid_visualizers]
+            if invalid:
+                raise ValueError(f"Invalid visualizer(s) specified: {invalid}. Valid options are: {valid_visualizers}")
+
+        # Store visualizer setting for later use
+        # Convert empty list to None for consistency
+        self._visualizer = visualizers if visualizers and len(visualizers) > 0 else None
+
+        # Check if both headless and visualizer are specified
+        if self._headless and self._visualizer is not None:
+            print(
+                f"[WARN][AppLauncher]: Both headless mode and visualizer(s) {self._visualizer} were specified."
+                " Headless mode is enabled, so no visualizers will be initialized."
+            )
+
     def _resolve_camera_settings(self, launcher_args: dict):
         """Resolve camera related settings."""
         enable_cameras_env = int(os.environ.get("ENABLE_CAMERAS", 0))
@@ -834,6 +928,14 @@ def _load_extensions(self):
         # for example: the `Camera` sensor class
         carb_settings_iface.set_bool("/isaaclab/render/rtx_sensors", False)
 
+        # set carb setting to store the visualizer backend(s) specified via command-line
+        # this allows SimulationContext to filter visualizers based on the --visualizer flag
+        # Store as comma-separated string for carb settings compatibility
+        if self._visualizer is not None:
+            carb_settings_iface.set_string("/isaaclab/visualizer", ",".join(self._visualizer))
+        else:
+            carb_settings_iface.set_string("/isaaclab/visualizer", "")
+
         # set fabric update flag to disable updating transforms when rendering is disabled
         carb_settings_iface.set_bool("/physics/fabricUpdateTransformations", self._rendering_enabled())