# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause
"""Keyboard controller for SE(3) control."""
import numpy as np
import torch
import weakref
from collections.abc import Callable
from dataclasses import dataclass
from scipy.spatial.transform import Rotation
import carb
import omni
from ..device_base import DeviceBase, DeviceCfg
@dataclass
class Se3KeyboardCfg(DeviceCfg):
"""Configuration for SE3 keyboard devices."""
pos_sensitivity: float = 0.4
rot_sensitivity: float = 0.8
retargeters: None = None
[docs]class Se3Keyboard(DeviceBase):
"""A keyboard controller for sending SE(3) commands as delta poses and binary command (open/close).
This class is designed to provide a keyboard controller for a robotic arm with a gripper.
It uses the Omniverse keyboard interface to listen to keyboard events and map them to robot's
task-space commands.
The command comprises of two parts:
* delta pose: a 6D vector of (x, y, z, roll, pitch, yaw) in meters and radians.
* gripper: a binary command to open or close the gripper.
Key bindings:
============================== ================= =================
Description Key (+ve axis) Key (-ve axis)
============================== ================= =================
Toggle gripper (open/close) K
Move along x-axis W S
Move along y-axis A D
Move along z-axis Q E
Rotate along x-axis Z X
Rotate along y-axis T G
Rotate along z-axis C V
============================== ================= =================
.. seealso::
The official documentation for the keyboard interface: `Carb Keyboard Interface <https://docs.omniverse.nvidia.com/dev-guide/latest/programmer_ref/input-devices/keyboard.html>`__.
"""
[docs] def __init__(self, cfg: Se3KeyboardCfg):
"""Initialize the keyboard layer.
Args:
cfg: Configuration object for keyboard settings.
"""
# store inputs
self.pos_sensitivity = cfg.pos_sensitivity
self.rot_sensitivity = cfg.rot_sensitivity
self._sim_device = cfg.sim_device
# acquire omniverse interfaces
self._appwindow = omni.appwindow.get_default_app_window()
self._input = carb.input.acquire_input_interface()
self._keyboard = self._appwindow.get_keyboard()
# note: Use weakref on callbacks to ensure that this object can be deleted when its destructor is called.
self._keyboard_sub = self._input.subscribe_to_keyboard_events(
self._keyboard,
lambda event, *args, obj=weakref.proxy(self): obj._on_keyboard_event(event, *args),
)
# bindings for keyboard to command
self._create_key_bindings()
# command buffers
self._close_gripper = False
self._delta_pos = np.zeros(3) # (x, y, z)
self._delta_rot = np.zeros(3) # (roll, pitch, yaw)
# dictionary for additional callbacks
self._additional_callbacks = dict()
def __del__(self):
"""Release the keyboard interface."""
self._input.unsubscribe_from_keyboard_events(self._keyboard, self._keyboard_sub)
self._keyboard_sub = None
def __str__(self) -> str:
"""Returns: A string containing the information of joystick."""
msg = f"Keyboard Controller for SE(3): {self.__class__.__name__}\n"
msg += f"\tKeyboard name: {self._input.get_keyboard_name(self._keyboard)}\n"
msg += "\t----------------------------------------------\n"
msg += "\tToggle gripper (open/close): K\n"
msg += "\tMove arm along x-axis: W/S\n"
msg += "\tMove arm along y-axis: A/D\n"
msg += "\tMove arm along z-axis: Q/E\n"
msg += "\tRotate arm along x-axis: Z/X\n"
msg += "\tRotate arm along y-axis: T/G\n"
msg += "\tRotate arm along z-axis: C/V"
return msg
"""
Operations
"""
[docs] def reset(self):
# default flags
self._close_gripper = False
self._delta_pos = np.zeros(3) # (x, y, z)
self._delta_rot = np.zeros(3) # (roll, pitch, yaw)
[docs] def add_callback(self, key: str, func: Callable):
"""Add additional functions to bind keyboard.
A list of available keys are present in the
`carb documentation <https://docs.omniverse.nvidia.com/dev-guide/latest/programmer_ref/input-devices/keyboard.html>`__.
Args:
key: The keyboard button to check against.
func: The function to call when key is pressed. The callback function should not
take any arguments.
"""
self._additional_callbacks[key] = func
[docs] def advance(self) -> torch.Tensor:
"""Provides the result from keyboard event state.
Returns:
torch.Tensor: A 7-element tensor containing:
- delta pose: First 6 elements as [x, y, z, rx, ry, rz] in meters and radians.
- gripper command: Last element as a binary value (+1.0 for open, -1.0 for close).
"""
# convert to rotation vector
rot_vec = Rotation.from_euler("XYZ", self._delta_rot).as_rotvec()
# return the command and gripper state
gripper_value = -1.0 if self._close_gripper else 1.0
delta_pose = np.concatenate([self._delta_pos, rot_vec])
command = np.append(delta_pose, gripper_value)
return torch.tensor(command, dtype=torch.float32, device=self._sim_device)
"""
Internal helpers.
"""
def _on_keyboard_event(self, event, *args, **kwargs):
"""Subscriber callback to when kit is updated.
Reference:
https://docs.omniverse.nvidia.com/dev-guide/latest/programmer_ref/input-devices/keyboard.html
"""
# apply the command when pressed
if event.type == carb.input.KeyboardEventType.KEY_PRESS:
if event.input.name == "L":
self.reset()
if event.input.name == "K":
self._close_gripper = not self._close_gripper
elif event.input.name in ["W", "S", "A", "D", "Q", "E"]:
self._delta_pos += self._INPUT_KEY_MAPPING[event.input.name]
elif event.input.name in ["Z", "X", "T", "G", "C", "V"]:
self._delta_rot += self._INPUT_KEY_MAPPING[event.input.name]
# remove the command when un-pressed
if event.type == carb.input.KeyboardEventType.KEY_RELEASE:
if event.input.name in ["W", "S", "A", "D", "Q", "E"]:
self._delta_pos -= self._INPUT_KEY_MAPPING[event.input.name]
elif event.input.name in ["Z", "X", "T", "G", "C", "V"]:
self._delta_rot -= self._INPUT_KEY_MAPPING[event.input.name]
# additional callbacks
if event.type == carb.input.KeyboardEventType.KEY_PRESS:
if event.input.name in self._additional_callbacks:
self._additional_callbacks[event.input.name]()
# since no error, we are fine :)
return True
def _create_key_bindings(self):
"""Creates default key binding."""
self._INPUT_KEY_MAPPING = {
# toggle: gripper command
"K": True,
# x-axis (forward)
"W": np.asarray([1.0, 0.0, 0.0]) * self.pos_sensitivity,
"S": np.asarray([-1.0, 0.0, 0.0]) * self.pos_sensitivity,
# y-axis (left-right)
"A": np.asarray([0.0, 1.0, 0.0]) * self.pos_sensitivity,
"D": np.asarray([0.0, -1.0, 0.0]) * self.pos_sensitivity,
# z-axis (up-down)
"Q": np.asarray([0.0, 0.0, 1.0]) * self.pos_sensitivity,
"E": np.asarray([0.0, 0.0, -1.0]) * self.pos_sensitivity,
# roll (around x-axis)
"Z": np.asarray([1.0, 0.0, 0.0]) * self.rot_sensitivity,
"X": np.asarray([-1.0, 0.0, 0.0]) * self.rot_sensitivity,
# pitch (around y-axis)
"T": np.asarray([0.0, 1.0, 0.0]) * self.rot_sensitivity,
"G": np.asarray([0.0, -1.0, 0.0]) * self.rot_sensitivity,
# yaw (around z-axis)
"C": np.asarray([0.0, 0.0, 1.0]) * self.rot_sensitivity,
"V": np.asarray([0.0, 0.0, -1.0]) * self.rot_sensitivity,
}
