Skip to content

Update installation and import for veRL integration #58

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ChiYeungLaw merged 3 commits into from
Apr 7, 2026
Merged

Update installation and import for veRL integration #58

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a545035
update installation and import: move fully async mode imports to try-…
StevenShi-23 Mar 23, 2026
d06d63e
update RL depdendency
StevenShi-23 Apr 6, 2026
7537a3e
Support new ray configuration layout in init_ray and run_ppo functions
StevenShi-23 Apr 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 19 additions & 16 deletions mcpuniverse/rl/integrations/verl/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,6 @@
from .mcp_reward_manager import MCPRewardManager
from .mcp_dataset import MCPDataset, create_mcp_dataset, mcp_collate_fn

# Fully async mode
from .fully_async import (
MCPFullyAsyncRollouter,
MCPFullyAsyncTrainer,
MCPAsyncTaskRunner,
MCPRolloutSample,
assemble_mcp_training_batch,
)

__all__ = [
# Core components
"MCPGeneratorInput",
Expand All @@ -38,11 +29,23 @@

# Hybrid mode trainer
"MCPPPOTrainer",

# Fully async mode
"MCPFullyAsyncRollouter",
"MCPFullyAsyncTrainer",
"MCPAsyncTaskRunner",
"MCPRolloutSample",
"assemble_mcp_training_batch",
]

# Fully async mode (optional — module may not exist yet)
try:
from .fully_async import (
MCPFullyAsyncRollouter,
MCPFullyAsyncTrainer,
MCPAsyncTaskRunner,
MCPRolloutSample,
assemble_mcp_training_batch,
)
__all__ += [
"MCPFullyAsyncRollouter",
"MCPFullyAsyncTrainer",
"MCPAsyncTaskRunner",
"MCPRolloutSample",
"assemble_mcp_training_batch",
]
except ImportError:
pass
Comment on lines +50 to +51

Copilot AI Mar 23, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The except ImportError: pass will also swallow import errors coming from inside fully_async once that module exists (e.g., missing optional deps, syntax errors), making failures hard to debug. Consider only handling the specific case where the submodule is missing (e.g., catch ModuleNotFoundError and verify e.name matches this package’s fully_async module, or pre-check with importlib.util.find_spec), and let other import errors surface.

Suggested change
except ImportError:
pass
except ModuleNotFoundError as e:
# Only ignore the error if the optional fully_async module itself is missing.
# Re-raise if some other module is missing so real import errors are not hidden.
if e.name not in (__name__ + ".fully_async", "fully_async"):
raise

Copilot uses AI. Check for mistakes.
9 changes: 8 additions & 1 deletion mcpuniverse/rl/integrations/verl/hybrid/mcp_main_ppo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,14 @@ def run_ppo(config) -> None:
ray.cancel(task)
raise

timeline_json_file = config.ray_init.get("timeline_json_file", None)
# veRL 0.7+ moved ray_init under ray_kwargs; support both layouts
if hasattr(config, "ray_init"):
ray_cfg = config.ray_init
elif hasattr(config, "ray_kwargs") and hasattr(config.ray_kwargs, "ray_init"):
ray_cfg = config.ray_kwargs.ray_init
else:
ray_cfg = {}
timeline_json_file = ray_cfg.get("timeline_json_file", None) if hasattr(ray_cfg, "get") else None
if timeline_json_file:
ray.timeline(filename=timeline_json_file)

Expand Down
21 changes: 19 additions & 2 deletions mcpuniverse/rl/integrations/verl/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ def init_ray(config, *, clean_tiktoken_cache: bool = False) -> None:
return

# -- env vars ----------------------------------------------------------
vllm_v1 = os.environ.get("VLLM_USE_V1", "0")
vllm_v1 = os.environ.get("VLLM_USE_V1", "1")
PPO_RAY_RUNTIME_ENV["env_vars"].update({"VLLM_USE_V1": vllm_v1})

tiktoken_cache_dir = os.environ.get("TIKTOKEN_CACHE_DIR", "/tmp/tiktoken-rs-cache")
Expand All @@ -134,6 +134,16 @@ def init_ray(config, *, clean_tiktoken_cache: bool = False) -> None:
if os.environ.get("LD_LIBRARY_PATH"):
PPO_RAY_RUNTIME_ENV["env_vars"]["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH"]

# Propagate NCCL env vars to Ray workers (e.g. NCCL_SHM_DISABLE for small /dev/shm)
nccl_keys = (
"NCCL_SHM_DISABLE", "NCCL_P2P_LEVEL", "NCCL_DEBUG",
"NCCL_NVLS_ENABLE", "NCCL_CUMEM_ENABLE",
)
for nccl_key in nccl_keys:
nccl_val = os.environ.get(nccl_key)
if nccl_val:
PPO_RAY_RUNTIME_ENV["env_vars"][nccl_key] = nccl_val

wandb_api_key = os.environ.get("WANDB_API_KEY")
if wandb_api_key:
PPO_RAY_RUNTIME_ENV["env_vars"]["WANDB_API_KEY"] = wandb_api_key
Expand All @@ -143,7 +153,14 @@ def init_ray(config, *, clean_tiktoken_cache: bool = False) -> None:

# -- connect / start ---------------------------------------------------
ray_address = os.environ.get("RAY_ADDRESS")
num_cpus = config.ray_init.get("num_cpus", None)
# veRL 0.7+ moved ray_init under ray_kwargs; support both layouts
if hasattr(config, "ray_init"):
ray_cfg = config.ray_init
elif hasattr(config, "ray_kwargs") and hasattr(config.ray_kwargs, "ray_init"):
ray_cfg = config.ray_kwargs.ray_init
else:
ray_cfg = {}
num_cpus = ray_cfg.get("num_cpus", None) if hasattr(ray_cfg, "get") else None

if ray_address:
logger.info(f"Connecting to existing Ray cluster at {ray_address}")
Expand Down
14 changes: 14 additions & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,20 @@ sglang = [
"flash-attn>=2.7.0",
"torch>=2.6.0",
]
rl = [
"verl @ git+https://github.com/verl-project/verl.git@9433f8a8f2771256ea4f8f94e4401bcfe9703228",
"torch>=2.6.0",
"ray>=2.0.0",
"tensordict>=0.6.0",
"hydra-core>=1.3.0",
"tqdm>=4.60.0",
"transformers>=4.40.0",
"numpy>=1.24.0",
"accelerate>=1.0.0",
"peft>=0.10.0",
"wandb>=0.15.0",
"datasets>=4.0.0",
]
Comment on lines +117 to +130

Copilot AI Mar 23, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This adds an rl extra in pyproject.toml, but the repo’s main install path documented in README.md is pip install -r requirements.txt, and requirements.txt currently only mentions the vllm/sglang extras. To avoid users still hitting missing-dependency errors for the VERL integration, consider updating the documented installation flow (e.g., mention pip install -e '.[rl]') and/or keeping requirements.txt guidance in sync with this new extra.

Copilot uses AI. Check for mistakes.
# System dependency: Node.js/npx is required for some MCP servers
# (google-maps, slack, notion, postgres, etc.)
# Install via: conda install -c conda-forge nodejs
Expand Down
Loading