Issue with 24_mujoco_interactive_simulator.py on Quest3s #95
Description
Issue with 24_mujoco_interactive_simulator.py on Quest3s
Description
I'm encountering an issue when trying to access the interactive MuJoCo simulation from my Quest3s browser. The simulation runs correctly and is accessible via a web browser on my Ubuntu machine at https://localhost:8012, but fails to load correctly on the Quest3s.
Steps to Reproduce
- Set up TLS for
localhostusing a self-signed certificate. I followed the Let's Encrypt guide for localhost certificates to generatecert.pemandkey.pem. - Use
adb reverse tcp:8012 tcp:8012to forward the port from my PC to the Quest3s. - Run the
24_mujoco_interactive_simulator.pyscript. - Open the Meta Quest browser and navigate to
https://localhost:8012.
Expected Behavior
The MuJoCo simulation should render correctly in the Quest3s browser, mirroring the behavior observed on the Ubuntu Firefox browser.
Actual Behavior
The Quest3s browser displays only a red dot at the center of the screen and blue grid.
Code
import os
from cmx import doc
from contextlib import nullcontext
MAKE_DOCS = os.getenv("MAKE_DOCS", None)
doc @ """
# Using in-browser mujoco as an interactive simulator for robots
This example shows you how to use the MuJoCo component in Vuer to create an interactive simulation of a car model.
"""
with doc, doc.skip if MAKE_DOCS else nullcontext():
from asyncio import sleep, Queue, create_task
from pathlib import Path
import numpy as np
from vuer import Vuer
from vuer.events import MjStep, Set, MjRender
from vuer.schemas import DefaultScene, SceneBackground, MjCameraView
from vuer.schemas import MuJoCo
from PIL import Image as PImage
from io import BytesIO
import cv2
app = Vuer(host='0.0.0.0', port=8012, cert='./config/cert.pem', key="./config/key.pem",
static_root=f"{Path(__file__).parent}/../_static")
asset_pref = "https://localhost:8012/static/"
# asset_pref = "https://docs.vuer.ai/en/latest/_static/"
fileName = "mujoco_scenes/car/car.mjcf.xml"
IS_MUJOCO_LOAD = True
@app.add_handler("ON_MUJOCO_LOAD")
async def handler(event, session):
global IS_MUJOCO_LOAD
IS_MUJOCO_LOAD = True
print("MuJoCo component loaded successfully.")
result_queue = Queue()
async def process_results():
while True:
result = await (await result_queue.get())
try:
if hasattr(result, "value") and isinstance(result.value, dict):
frame = result.value.get("depthFrame") or result.value.get("frame")
if frame:
pil_image = PImage.open(BytesIO(frame))
img = np.array(pil_image)
img_bgr = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
cv2.imshow("monitor", img_bgr)
if cv2.waitKey(1) == ord("q"):
exit()
else:
print("No valid frame data found in result.")
else:
print("Invalid result structure or missing 'value' attribute.")
except Exception as e:
print(f"Error processing result: {e}")
# use `start=True` to start the app immediately
@app.spawn(start=True)
async def main(session):
session @ Set(
DefaultScene(
SceneBackground(),
MuJoCo(
key="simple",
src=asset_pref + fileName,
pause=False,
useLights=True,
unpauseOnDrag=False, # Whether to unpause the simulation when dragging the object
dragForceScale=1.0, # Scale of the drag force applied to the object when dragging
showDragArrow=True, # Whether to show the drag arrow when dragging the object
showDragForceText=True, # Whether to show the drag force text when dragging the object
),
MjCameraView(
key="cam1",
position=[0, 0.35, 0.5],
rotation=[-0.4, 0, 0],
width=640,
height=480,
distanceToCamera=0.1,
movable=True, # Whether the camera can be moved by the user
showCameraFrustum=True, # Whether to show the camera in the scene
),
up=[0, 1, 0],
show_helper=False,
),
)
await sleep(2)
create_task(process_results())
i = 0
while True:
i += 1
if not IS_MUJOCO_LOAD:
print("Waiting for MuJoCo component to load...")
await sleep(1)
continue
await session.rpc(
MjStep(key="simple", sim_steps=10, ctrl=[1, -1]),
ttl=5,
)
"""
Instead of waiting for the rendering result, we can use a queue to handle the
rendering tasks as follows:
"""
with doc:
promise = session.rpc(MjRender(key="cam1"))
await result_queue.put(promise)
await sleep(1 / 30)
doc @ """
Alternatively, you can use
```python
result = await session.rpc(MjRender(key="cam1"), ttl=5)
try:
if hasattr(result, "value") and isinstance(result.value, dict):
frame = result.value.get("depthFrame") or result.value.get("frame")
if frame:
pil_image = PImage.open(BytesIO(frame))
img = np.array(pil_image)
img_bgr = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
cv2.imshow("monitor", img_bgr)
if cv2.waitKey(1) == ord("q"):
exit()
else:
print("No valid frame data found in result.")
else:
print("Invalid result structure or missing 'value' attribute.")
except Exception as e:
print(f"Error processing result: {e}")
to waiting for the rendering result directly, but using a queue is more efficient for handling multiple rendering tasks.
"""
doc.flush()
Terminal Log
$ python3 24_mujoco_interactive_simulator.py
ML_LOGGER_USER is not set. This is required for online usage.
File output at file:///home/vuer/docs/examples/24_mujoco_interactive_simulator.md
Serving file:///home/vuer/docs/_static at /static
Visit: https://vuer.ai
websocket is connected. id:bc1753a8-3115-4719-84e8-2960703c88ac
default socket worker is up, adding clientEvents
Uplink task running. id:bc1753a8-3115-4719-84e8-2960703c88ac
websocket is now disconnected. Removing the socket.
WebSocket connection closed
uplink:bc1753a8-3115-4719-84e8-2960703c88ac is not in websocket pool
websocket is connected. id:71ef1999-cf34-4593-84b6-c84471f6c3ae
default socket worker is up, adding clientEvents
Uplink task running. id:71ef1999-cf34-4593-84b6-c84471f6c3ae
Task exception was never retrieved
future: <Task finished name='Task-39' coro=<Vuer.downlink.<locals>.handler() done, defined at /home/vuer/.pixi/envs/default/lib/python3.10/site-packages/vuer/server.py:661> exception=TimeoutError()>
Traceback (most recent call last):
File "/home/vuer/.pixi/envs/default/lib/python3.10/asyncio/locks.py", line 213, in wait
await fut
asyncio.exceptions.CancelledError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/vuer/.pixi/envs/default/lib/python3.10/asyncio/tasks.py", line 458, in wait_for
fut.result()
asyncio.exceptions.CancelledError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/vuer/.pixi/envs/default/lib/python3.10/site-packages/vuer/server.py", line 667, in handler
raise e
File "/home/vuer/.pixi/envs/default/lib/python3.10/site-packages/vuer/server.py", line 663, in handler
await self.socket_handler(vuer_proxy)
File "/home/vuer/docs/examples/24_mujoco_interactive_simulator.py", line 107, in main
await session.rpc(
File "/home/vuer/.pixi/envs/default/lib/python3.10/site-packages/vuer/server.py", line 154, in rpc
raise e
File "/home/vuer/.pixi/envs/default/lib/python3.10/site-packages/vuer/server.py", line 151, in rpc
await asyncio.wait_for(rpc_event.wait(), ttl)
File "/home/vuer/.pixi/envs/default/lib/python3.10/asyncio/tasks.py", line 460, in wait_for
raise exceptions.TimeoutError() from exc
asyncio.exceptions.TimeoutError
uplink:71ef1999-cf34-4593-84b6-c84471f6c3ae is not in websocket pool
websocket is now disconnected. Removing the socket.
WebSocket connection closed
Task exception was never retrieved
future: <Task finished name='Task-46' coro=<Vuer.downlink.<locals>.handler() done, defined at /home/vuer/.pixi/envs/default/lib/python3.10/site-packages/vuer/server.py:661> exception=TimeoutError()>
Traceback (most recent call last):
File "/home/vuer/.pixi/envs/default/lib/python3.10/asyncio/locks.py", line 213, in wait
await fut
asyncio.exceptions.CancelledError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/vuer/.pixi/envs/default/lib/python3.10/asyncio/tasks.py", line 458, in wait_for
fut.result()
asyncio.exceptions.CancelledError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/vuer/.pixi/envs/default/lib/python3.10/site-packages/vuer/server.py", line 667, in handler
raise e
File "/home/vuer/.pixi/envs/default/lib/python3.10/site-packages/vuer/server.py", line 663, in handler
await self.socket_handler(vuer_proxy)
File "/home/vuer/docs/examples/24_mujoco_interactive_simulator.py", line 107, in main
await session.rpc(
File "/home/vuer/.pixi/envs/default/lib/python3.10/site-packages/vuer/server.py", line 154, in rpc
raise e
File "/home/vuer/.pixi/envs/default/lib/python3.10/site-packages/vuer/server.py", line 151, in rpc
await asyncio.wait_for(rpc_event.wait(), ttl)
File "/home/vuer/.pixi/envs/default/lib/python3.10/asyncio/tasks.py", line 460, in wait_for
raise exceptions.TimeoutError() from exc
asyncio.exceptions.TimeoutError
websocket is connected. id:394da1f6-0942-4f50-992f-657561956638
default socket worker is up, adding clientEvents
Uplink task running. id:394da1f6-0942-4f50-992f-657561956638
^CTraceback (most recent call last):
File "/home/vuer/docs/examples/24_mujoco_interactive_simulator.py", line 67, in <module>
async def main(session):
File "/home/vuer/.pixi/envs/default/lib/python3.10/site-packages/vuer/server.py", line 500, in wrap_fn
self.run()
File "/home/vuer/.pixi/envs/default/lib/python3.10/site-packages/vuer/server.py", line 849, in run
super().run()
File "/home/vuer/.pixi/envs/default/lib/python3.10/site-packages/vuer/base.py", line 137, in run
event_loop.run_forever()
File "/home/vuer/.pixi/envs/default/lib/python3.10/asyncio/base_events.py", line 595, in run_forever
self._run_once()
File "/home/vuer/.pixi/envs/default/lib/python3.10/asyncio/base_events.py", line 1864, in _run_once
ntodo = len(self._ready)
KeyboardInterrupt
pixi env
# pixi.toml
[dependencies]
python = "==3.10"
setuptools = "<80.0"
aiohttp = "==3.10.5"
pandas = ">=2.3.0,<3"
[pypi-dependencies]
vuer = { version = "==0.0.67", extras = ["all"] }
cmx = ">=0.0.46, <0.0.47"
opencv-python = ">=4.12.0.88, <5"
ml-logger = ">=0.10.24, <0.11"
aiohttp-cors = ">=0.8.1, <0.9"
killport = ">=1.2.0, <2"
functional-notations = ">=0.5.2, <0.6"
$ pixi tree vuer
└── vuer 0.0.67
├── params_proto 2.13.2
│ ├── waterbear 2.6.8
│ ├── argparse 1.4.0
│ ├── argcomplete 3.6.2
│ ├── expandvars 1.1.2
│ └── termcolor 3.1.0
├── pillow 11.3.0
├── msgpack 1.1.1
├── numpy 2.2.6
│ ├── __glibc
│ ├── libblas 3.9.0
│ │ └── libopenblas 0.3.29
│ │ ├── __glibc (*)
│ │ ├── libgcc 15.1.0
│ │ │ ├── __glibc (*)
│ │ │ └── _openmp_mutex 4.5
│ │ │ ├── _libgcc_mutex 0.1
│ │ │ └── libgomp 15.1.0
│ │ │ └── __glibc (*)
│ │ ├── libgfortran 15.1.0
│ │ │ └── libgfortran5 15.1.0
│ │ │ ├── __glibc (*)
│ │ │ └── libgcc 15.1.0 (*)
│ │ └── libgfortran5 15.1.0 (*)
│ ├── libcblas 3.9.0
│ │ └── libblas 3.9.0 (*)
│ ├── libgcc 15.1.0 (*)
│ ├── liblapack 3.9.0
│ │ └── libblas 3.9.0 (*)
│ ├── libstdcxx 15.1.0
│ │ ├── __glibc (*)
│ │ └── libgcc 15.1.0 (*)
│ ├── python 3.10.0
│ │ ├── bzip2 1.0.8
│ │ │ ├── __glibc (*)
│ │ │ └── libgcc-ng 15.1.0
│ │ │ └── libgcc 15.1.0 (*)
│ │ ├── ld_impl_linux-64 2.43
│ │ │ └── __glibc (*)
│ │ ├── libffi 3.4.6
│ │ │ ├── __glibc (*)
│ │ │ └── libgcc 15.1.0 (*)
│ │ ├── libgcc-ng 15.1.0 (*)
│ │ ├── libnsl 2.0.1
│ │ │ └── libgcc-ng 15.1.0 (*)
│ │ ├── libuuid 2.38.1
│ │ │ └── libgcc-ng 15.1.0 (*)
│ │ ├── libzlib 1.2.13
│ │ │ └── libgcc-ng 15.1.0 (*)
│ │ ├── ncurses 6.5
│ │ │ ├── __glibc (*)
│ │ │ └── libgcc 15.1.0 (*)
│ │ ├── openssl 3.5.0
│ │ │ ├── __glibc (*)
│ │ │ ├── ca-certificates 2025.4.26
│ │ │ │ └── __unix
│ │ │ └── libgcc 15.1.0 (*)
│ │ ├── readline 8.2
│ │ │ ├── libgcc 15.1.0 (*)
│ │ │ └── ncurses 6.5 (*)
│ │ ├── sqlite 3.46.0
│ │ │ ├── libgcc-ng 15.1.0 (*)
│ │ │ ├── libsqlite 3.46.0
│ │ │ │ ├── libgcc-ng 15.1.0 (*)
│ │ │ │ └── libzlib 1.2.13 (*)
│ │ │ ├── libzlib 1.2.13 (*)
│ │ │ ├── ncurses 6.5 (*)
│ │ │ └── readline 8.2 (*)
│ │ ├── tk 8.6.13
│ │ │ ├── libgcc-ng 15.1.0 (*)
│ │ │ └── libzlib 1.2.13 (*)
│ │ ├── tzdata 2025b
│ │ └── xz 5.8.1
│ │ ├── __glibc (*)
│ │ ├── libgcc 15.1.0 (*)
│ │ ├── liblzma 5.8.1
│ │ │ ├── __glibc (*)
│ │ │ └── libgcc 15.1.0 (*)
│ │ ├── liblzma-devel 5.8.1
│ │ │ ├── __glibc (*)
│ │ │ ├── libgcc 15.1.0 (*)
│ │ │ └── liblzma 5.8.1 (*)
│ │ ├── xz-gpl-tools 5.8.1
│ │ │ ├── __glibc (*)
│ │ │ ├── libgcc 15.1.0 (*)
│ │ │ └── liblzma 5.8.1 (*)
│ │ └── xz-tools 5.8.1
│ │ ├── __glibc (*)
│ │ ├── libgcc 15.1.0 (*)
│ │ └── liblzma 5.8.1 (*)
│ └── python_abi 3.10
└── websockets 15.0.1
Additional Information
The local web server is confirmed to be running and accessible from the Ubuntu machine.
The port forwarding via adb is confirmed to be working.
All connections within the script are configured to use HTTPS and wss.