diff --git a/README.md b/README.md index 81fb646d6e..003a4fff78 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ + # MLPerf® Inference Benchmark Suite MLPerf Inference is a benchmark suite for measuring how fast systems can run models in a variety of deployment scenarios. diff --git a/docs/README.md b/docs/README.md index 496a93718a..8f888f148c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,7 @@ # Documentation Website for MLPerf Inference using the MLC interface ## Commands to get the website running locally + ``` git clone https://github.com/mlcommons/inference cd inference diff --git a/docs/benchmarks/image_classification/resnet50.md b/docs/benchmarks/image_classification/resnet50.md index 3840e6be6b..e2619d95cc 100644 --- a/docs/benchmarks/image_classification/resnet50.md +++ b/docs/benchmarks/image_classification/resnet50.md @@ -10,14 +10,15 @@ hide: === "MLCommons-Python" ## MLPerf Reference Implementation in Python -{{ mlperf_inference_implementation_readme (4, "resnet50", "reference") }} +{{ mlperf_inference_implementation_readme (4, "resnet50", "reference", categories=["Edge"]) }} === "Nvidia" ## Nvidia MLPerf Implementation {{ mlperf_inference_implementation_readme (4, "resnet50", "nvidia") }} - + +{# === "Intel" ## Intel MLPerf Implementation @@ -34,4 +35,4 @@ hide: {{ mlperf_inference_implementation_readme (4, "resnet50", "cpp") }} ---> +#} diff --git a/docs/benchmarks/language/bert.md b/docs/benchmarks/language/bert.md index 51ac91c86e..9d0ae09b25 100644 --- a/docs/benchmarks/language/bert.md +++ b/docs/benchmarks/language/bert.md @@ -8,9 +8,9 @@ hide: === "MLCommons-Python" ## MLPerf Reference Implementation in Python -{{ mlperf_inference_implementation_readme (4, "bert-99", "reference") }} +{{ mlperf_inference_implementation_readme (4, "bert-99", "reference", categories=["Edge"]) }} -{{ mlperf_inference_implementation_readme (4, "bert-99.9", "reference") }} +{{ mlperf_inference_implementation_readme (4, "bert-99.9", "reference", categories=["Edge"]) }} === "Nvidia" ## Nvidia MLPerf Implementation @@ -19,7 +19,8 @@ hide: {{ mlperf_inference_implementation_readme (4, "bert-99.9", "nvidia") }} -s \ No newline at end of file + +#} \ No newline at end of file diff --git a/docs/benchmarks/language/llama2-70b.md b/docs/benchmarks/language/llama2-70b.md index effc7b18a6..7e4b243aa6 100644 --- a/docs/benchmarks/language/llama2-70b.md +++ b/docs/benchmarks/language/llama2-70b.md @@ -12,6 +12,8 @@ hide: {{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "reference") }} +{# + === "Nvidia" ## Nvidia MLPerf Implementation @@ -19,6 +21,7 @@ hide: {{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "nvidia") }} +#} + +#} diff --git a/docs/submission/submission-cli.md b/docs/submission/submission-cli.md index f920bbfa11..d573a029f1 100644 --- a/docs/submission/submission-cli.md +++ b/docs/submission/submission-cli.md @@ -9,35 +9,54 @@ Please refer to the [installation page](site:inference/install/) to install MLCF === "Custom automation based MLPerf results" If you have not followed the `mlcr` commands under the individual model pages in the [benchmarks](../index.md) directory, please make sure that the result directory is structured in the following way. You can see the real examples for the expected folder structure [here](https://github.com/mlcommons/inference/tree/submission-generation-examples). - ``` - └── System description ID(SUT Name) - ├── system_meta.json - └── Benchmark - └── Scenario - ├── Performance - | └── run_1 run for all scenarios - | ├── mlperf_log_summary.txt - | └── mlperf_log_detail.txt - ├── Accuracy - | ├── mlperf_log_summary.txt - | ├── mlperf_log_detail.txt - | ├── mlperf_log_accuracy.json - | └── accuracy.txt - |── Compliance_Test_ID - | ├── Performance - | | └── run_x/#1 run for all scenarios - | | ├── mlperf_log_summary.txt - | | └── mlperf_log_detail.txt - | ├── Accuracy # for TEST01 only - | | ├── baseline_accuracy.txt (if test fails in deterministic mode) - | | ├── compliance_accuracy.txt (if test fails in deterministic mode) - | | ├── mlperf_log_accuracy.json - | | └── accuracy.txt - | ├── verify_performance.txt - | └── verify_accuracy.txt # for TEST01 only - |── user.conf - └── measurements.json - ``` + + The submission generator supports two kinds of results, shown in the tabs below. **LoadGen based results** are produced by the reference/optimized implementations and contain the usual `mlperf_log_*` files. **Inference endpoint based results** are produced by the LLM/API endpoint harness; a scenario is treated as an endpoint run when a `config.yaml` (or `config.yml`) is present at the scenario root, in which case a single JSON summary replaces the LoadGen logs and there is no `run_1` subfolder. The mode folder names (`performance`, `accuracy`) must be lowercase. + + === "LoadGen based results" + ``` + └── System description ID(SUT Name) + ├── system_meta.json + └── Benchmark + └── Scenario + ├── performance + | └── run_1 run for all scenarios + | ├── mlperf_log_summary.txt + | └── mlperf_log_detail.txt + ├── accuracy + | ├── mlperf_log_summary.txt + | ├── mlperf_log_detail.txt + | ├── mlperf_log_accuracy.json + | └── accuracy.txt + |── Compliance_Test_ID + | ├── performance + | | └── run_x/#1 run for all scenarios + | | ├── mlperf_log_summary.txt + | | └── mlperf_log_detail.txt + | ├── accuracy # for TEST01 only + | | ├── baseline_accuracy.txt (if test fails in deterministic mode) + | | ├── compliance_accuracy.txt (if test fails in deterministic mode) + | | ├── mlperf_log_accuracy.json + | | └── accuracy.txt + | ├── verify_performance.txt + | └── verify_accuracy.txt # for TEST01 only + |── user.conf + └── measurements.json + ``` + + === "Inference endpoint based results" + ``` + └── System description ID(SUT Name) + ├── system_meta.json + └── Benchmark + └── Scenario + ├── config.yaml # endpoint run marker (config.yml also accepted); copied into the submission + ├── measurements.json + ├── performance + | └── result_summary.json + └── accuracy + └── accuracy_results.json + ``` + Note: endpoint runs do not require a `user.conf` and do not use the `run_1` subfolder or any `mlperf_log_*` files.
Click here if you are submitting in open division diff --git a/docs/system-info/index.md b/docs/system-info/index.md new file mode 100644 index 0000000000..02764c366c --- /dev/null +++ b/docs/system-info/index.md @@ -0,0 +1,308 @@ +# MLPerf Inference System Info Collection + +This guide covers how to automatically collect hardware and software information from one or more nodes for MLPerf Inference submissions using the MLC `get-mlperf-multi-node-system-info` script. + +!!! note "MLPerf Inference v6.1 scope" + For inference round v6.1, the sysinfo tool aims to automate system inventory collection (CPU, GPU, memory, storage, OS and software stack). Power and Network mode fields are scoped for future rounds and scaffolded as empty strings in the output. Submitters have to fill them in manually before submission. + +## Prerequisites + +**Install MLC** — Follow the [MLC installation guide](../install/index.md) to set up `mlc-scripts`. + +**SSH access** — For multi-node setups the script SSHes into each remote node to run the hardware probe. Password-based SSH works, but passwordless (key-based) access is strongly recommended — it is seamless and avoids repeated prompts when probing a large number of nodes. + +To set up passwordless SSH: + +```bash +# Generate a key if you don't have one +ssh-keygen -t rsa -b 4096 + +# Copy it to every target node +ssh-copy-id user@node1 +ssh-copy-id user@node2 + +# Verify +ssh user@node1 "hostname && nvidia-smi -L" +``` + +## Quick Start + +The same `get-mlperf-multi-node-system-info` script handles everything from a single node to a large cluster. Internally it always calls `get-mlperf-single-node-system-info` on each target node (including the local machine as node 0). When no `--ssh_ids` are given it probes only the local machine. + +**Single node (local machine)** + +```bash +mlcr get-mlperf-multi-node-system-info,_cuda,_inference \ + --system_name="My-1xH100-System" +``` + +**Multi-node cluster** + +```bash +mlcr get-mlperf-multi-node-system-info,_cuda,_inference \ + --ssh_ids="user@node1:22,user@node2:22,user@node3:22" \ + --system_name="24xH100-Cluster" +``` + +The local machine is included as node 0; each SSH target becomes node 1, 2, 3, and so on. To collect from SSH targets only and exclude the local machine: + +```bash +mlcr get-mlperf-multi-node-system-info,_cuda,_inference,_exclude_current_node \ + --ssh_ids="user@node1:22,user@node2:22" \ + --system_name="Remote-Only-Cluster" +``` + +## Using a Config File + +For repeated runs or shared team configs, store submission metadata in a YAML or JSON file instead of passing many CLI flags. CLI arguments always take precedence over config file values. + +```yaml +# system_config.yaml +submitter_org_names: "Your Organization" +system_name: "8xH100-vLLM-Server" +division: "open" +system_category: "datacenter" +system_availability_status: "available" +cooling: "air" +hw_notes: "DGX H100 node, 8x NVLink-connected H100 SXM5" +system_type_detail: "on-premise" +``` + +```bash +mlcr get-mlperf-multi-node-system-info,_cuda,_inference \ + --config_file=system_config.yaml \ + --ssh_ids="user@node1:22,user@node2:22" +``` + + + +## Serving Framework Detection + +`framework` is a required field for the MLPerf Inference submission. The script can detect it automatically in two ways. + +**Auto-detect via HTTP probe** — provide `--endpoint_url` and the script probes the running inference server: + +```bash +mlcr get-mlperf-multi-node-system-info,_cuda,_inference \ + --ssh_ids="user@node1:22" \ + --endpoint_url="http://node1:8000" \ + --system_name="vLLM-System" +``` + +| Framework | Endpoint probed | +|-----------|----------------| +| TRT-LLM | `/perf_metrics` | +| vLLM | `/version` | +| SGLang | `/get_server_info` | + + +## Output + +The script writes `system-info-multi-node.json` in the current directory. The full path is printed at the end of the run and exported as `MLC_MULTI_NODE_SYSTEM_INFO_FILE_PATH`. + +With `_inference`, the output is a flat JSON as required for the MLPerf Inference submission. Submitters must verify the generated `system-info-multi-node.json` and manually fill in any fields that are empty. Fields expected to be auto-detected (see [Hardware and Software Fields](#hardware-and-software-fields) below) should not be empty — if any of those come out as an empty string, please [raise an issue](https://github.com/mlcommons/mlperf-automations/issues) with the field name and details of your machine. + +When the `_network` variation is also active, all network mode fields are appended to the output as empty strings and must be filled in manually — see [Network Mode Fields](#network-mode-fields-with-_network-variation) for the full list. + +```json +{ + "submitter": "Your Organization", + "system_name": "2xDGX-H100-vLLM", + "status": "available", + "system_type": "datacenter", + "division": "open", + "system_size": "16x NVIDIA H100 80GB HBM3", + "number_of_nodes": 2, + "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 112, + "host_processor_vcpu_count": 224, + "host_processor_frequency": "3.80 GHz", + "host_processor_caches": "L1d: 4.4 MiB; L1i: 2.2 MiB; L2: 224 MiB; L3: 210 MiB", + "host_processor_interconnect": "", + "host_memory_capacity": "2.2T", + "host_storage_type": "NVMe SSD", + "host_storage_capacity": "1.8 TB SSD", + "host_memory_configuration": "DDR5", + "host_networking": "mlx5_0: native InfiniBand", + "host_networking_topology": "", + "host_network_card_count": "3x mlx5_0: native InfiniBand", + "accelerator_model_name": "NVIDIA H100 80GB HBM3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80GiB", + "accelerator_memory_configuration": "80 GiB HBM3", + "accelerator_host_interconnect": "PCIe Gen5 x16", + "accelerator_interconnect": "NVLink", + "accelerator_interconnect_topology": "", + "accelerator_frequency": "", + "accelerator_on-chip_memories": "Shared Memory: 228 KB/block", + "framework": "vLLM 0.4.3", + "operating_system": "ubuntu 24.04", + "other_software_stack": "CUDA 12.9, Driver 575.57.08", + "hw_notes": "", + "sw_notes": "", + "other_hardware": "", + "cooling": "air", + "system_type_detail": "" +} +``` + +## Information Captured + +### Hardware and Software Fields + +These are collected automatically on each node. If a value cannot be detected (driver missing, command unavailable), the field is set to `""` or `"N/A"` in the output for manual completion. + +| Field | Description | Auto-Detected | +|-------|-------------|:---:| +| `host_processor_model_name` | CPU model name | ✅ | +| `host_processors_per_node` | Number of CPU sockets | ✅ | +| `host_processor_core_count` | Physical CPU cores per socket | ✅ | +| `host_processor_frequency` | CPU maximum frequency | ✅ | +| `host_processor_caches` | L1d / L1i / L2 / L3 cache sizes | ✅ | +| `host_processor_interconnect` | CPU-to-CPU interconnect inferred from NUMA topology | ✅ | +| `host_memory_capacity` | Total system RAM | ✅ | +| `host_memory_configuration` | Memory type and speed | ✅ | +| `host_storage_type` | Primary storage type (NVMe, SSD, HDD) | ✅ | +| `host_storage_capacity` | Total disk capacity | ✅ | +| `host_networking` | Primary NIC description | ✅ | +| `host_network_card_count` | NIC count and model | ✅ | +| `accelerator_model_name` | GPU model name | ✅ | +| `accelerators_per_node` | Number of GPUs per node | ✅ | +| `accelerator_memory_capacity` | GPU memory per device | ✅ | +| `accelerator_memory_configuration` | GPU memory size and type | ✅ | +| `accelerator_host_interconnect` | Host-to-GPU link (PCIe Gen, NVLink) | ✅ | +| `accelerator_interconnect` | GPU-to-GPU link (NVLink, xGMI) | ✅ | +| `accelerator_interconnect_topology` | Interconnect topology description | ⚠️ CUDA only; may be empty | +| `accelerator_frequency` | GPU clock frequency | ✅ | +| `accelerator_on-chip_memories` | Shared memory per SM block | ✅ | +| `operating_system` | OS distribution and version | ✅ | +| `other_software_stack` | CUDA/ROCm version + driver version | ✅ | +| `number_of_nodes` | Total node count | ✅ Computed from SSH targets | +| `framework` | Inference framework and version | ⚠️ Requires `--endpoint_url` | + +### Submission Identity Fields + +These fields are not detectable from hardware and must be supplied via CLI flags, a config file, or by directly editing the output JSON before submission. + +| Field | CLI Flag | Config Key | Notes | +|-------|----------|-----------|-------| +| `system_name` | `--system_name` | `system_name` | Required | +| `submitter` | `--submitter_org_names` | `submitter_org_names` | Required | +| `division` | `--division` | `division` | `open` or `closed` | +| `status` | `--system_availability_status` | `system_availability_status` | System availability — `available` (publicly available), `preview` (available soon), or `rdi` (Research, Development, and Internal use only) | +| `system_type` | `--category` | `system_category` | `datacenter` or `edge` | +| `cooling` | `--cooling` | `cooling` | e.g. `air`, `liquid` | +| `hw_notes` | `--hw_notes` | `hw_notes` | Hardware notes | +| `sw_notes` | *(manual edit)* | — | Software notes; fill in the output JSON | +| `host_networking_topology` | *(manual edit)* | — | Network topology description (not auto-detected; different from `host_networking` which is captured automatically) | +| `system_type_detail` | `--system_type_detail` | `system_type_detail` | More specific system type — `cloud`, `on-premise`, `edge-server`, or `edge-device` (optional) | + +### Network Mode Fields (with `_network` variation) + +When the `_network` variation is active alongside `_inference`, the script adds the fields required for network mode submissions as per MLPerf Inference submission rules. All fields are initialised to `""` and must be filled in manually. + +`is_network`, `network_type`, `network_media`, `network_rate`, `nic_loadgen`, `number_nic_loadgen`, `net_software_stack_loadgen`, `network_protocol`, `number_connections`, `nic_sut`, `number_nic_sut`, `net_software_stack_sut`, `network_topology` + +### Power Measurement Fields (with `_power` variation) + +When the `_power` variation is active, the following additional fields are required. All are initialised to `""` and must be filled in manually. + +`power_management`, `filesystem`, `boot_firmware_version`, `management_firmware_version`, `other_hardware`, `number_of_type_nics_installed`, `nics_enabled_firmware`, `nics_enabled_os`, `nics_enabled_connected`, `network_speed_mbit`, `power_supply_quantity_and_rating_watts`, `power_supply_details`, `disk_drives`, `disk_controllers`, `system_power_only` + +### Optional Nameplate Power YAML (`_inference_optional_nameplate`, `_redfish`) + +Separate from the fields above, the script can also generate the **optional +nameplate / design-power YAML** described in the inference submission rules +(`tools/submission/submission_structure.md`). This is not part of +`system-info-multi-node.json` — it's a standalone file consumed by the +MLPerf Inference `submission_checker`'s `nameplate_power_check`, which sums +`PowerCapacityWatts` across the `Min PSUs Needed` largest PSUs per leaf node +of PSU declarations. The checker expects it at +`systems/_power.yaml` (required starting `v6.1`). + +Two modes, selected by whether `_redfish` is also active: + +| Tags | What gets written | +|------|--------------------| +| `_inference_optional_nameplate` alone | A generic **skeleton template** — placeholder `My Rack 1` / `My Server 1` / `My Switch 1` labels, two PSUs at 1200W each, `Description: 'Optional Description'` — for you to fill in by hand. No BMC is contacted. | +| `_inference_optional_nameplate,_redfish` | The **real** PSU nameplate/capacity data, queried live from a Redfish-enabled BMC (or a [DMTF Redfish mockup server](https://github.com/DMTF/Redfish-Mockup-Server) for local testing) | + +**Skeleton template (no BMC):** + +```bash +mlcr get-mlperf-multi-node-system-info,_inference,_inference_optional_nameplate \ + --system_name="My-System" +``` + +**Populated from a live Redfish BMC:** + +```bash +mlcr get-mlperf-multi-node-system-info,_inference,_redfish,_inference_optional_nameplate \ + --system_name="My-System" \ + --redfish_endpoint="https://bmc.example.com" \ + --redfish_username="admin" \ + --redfish_password="secret" +``` + +When populated from Redfish, PSU data is sourced as follows: + +| Nameplate field | Redfish source | +|---|---| +| `PSUs[].Name` / `PowerCapacityWatts` | `Chassis//PowerSubsystem/PowerSupplies/` (preferred), falling back to the legacy `Chassis//Power` → `PowerSupplies[]` on BMCs that only implement the older schema | +| `Min PSUs Needed` | `Chassis//PowerSubsystem` → `PowerSupplyRedundancy[].MinNeededInGroup`, when reported; otherwise conservatively defaults to the number of installed PSUs (no redundancy credit) | + +Redfish has no concept of rack/system grouping above a chassis — each +chassis becomes one flat leaf under ``, even with real BMC +data. If you want an explicit rack layer in between (as the skeleton +template shows), add it to the generated YAML by hand. + +The output is written to `_power.yaml` and its path is exported as +`MLC_NAMEPLATE_POWER_YAML_FILE_PATH`. As with the main system-info JSON, +you still need to copy/rename this file into your submission's `systems/` +directory to match whatever `` that submission actually +uses. + +## Available Variations + +| Variation | Description | +|-----------|-------------| +| `_cuda` | Probe NVIDIA GPUs via CUDA | +| `_rocm` | Probe AMD GPUs via ROCm | +| `_xpu` | Probe Intel GPUs via XPU | +| `_inference` | Flat JSON output as required for MLPerf Inference submission | +| `_exclude_current_node` | Skip the local machine; collect only from SSH targets | +| `_network` | Add network mode fields to the output | +| `_power` | Add power measurement fields to the output | +| `_redfish` | Capture live PSU/power data from a Redfish BMC (used with `_inference_optional_nameplate`, or alone for a raw reference capture) | +| `_inference_optional_nameplate` | Generate the optional nameplate power YAML — a skeleton template alone, or populated from Redfish when stacked with `_redfish` (see [Optional Nameplate Power YAML](#optional-nameplate-power-yaml-_inference_optional_nameplate-_redfish)) | + +Variations can be stacked: + +```bash +mlcr get-mlperf-multi-node-system-info,_cuda,_inference,_power \ + --ssh_ids="user@node1:22" \ + --system_name="My-System" +``` + +## Key Parameters Reference + +| CLI Flag | Environment Variable | Description | +|----------|---------------------|-------------| +| `--ssh_ids` | `MLC_MULTINODE_SYSTEM_SSH_IDS` | Comma-separated SSH targets (`user@host:port`) | +| `--system_name` | `MLC_MLPERF_SYSTEM_NAME` | System identifier (required) | +| `--config_file` | `MLC_MLPERF_CONFIG_FILE` | Path to a JSON / YAML config file | +| `--endpoint_url` | `MLC_MLPERF_ENDPOINT_URL` | Endpoint URL for serving framework auto-detection | +| `--submitter_org_names` | `MLC_MLPERF_SUBMITTER` | Submitting organization name | +| `--division` | `MLC_MLPERF_SUBMISSION_DIVISION` | `open` or `closed` | +| `--category` | `MLC_MLPERF_SUBMISSION_SYSTEM_TYPE` | `datacenter` or `edge` | +| `--system_availability_status` | `MLC_MLPERF_SUBMISSION_SYSTEM_STATUS` | `available`, `preview`, or `rdi` | +| `--cooling` | `MLC_MLPERF_COOLING` | Cooling method | +| `--hw_notes` | `MLC_MLPERF_HARDWARE_NOTES` | Hardware notes | +| `--system_type_detail` | `MLC_MLPERF_SYSTEM_TYPE_DETAIL` | `cloud`, `on-premise`, `edge-server`, or `edge-device` (optional) | +| `--redfish_endpoint` | `MLC_REDFISH_ENDPOINT` | Redfish BMC base URL (only relevant with `_redfish`) | +| `--redfish_username` | `MLC_REDFISH_USERNAME` | BMC username; leave unset for an unauthenticated mockup | +| `--redfish_password` | `MLC_REDFISH_PASSWORD` | BMC password | + +If you hit any issues while using this script, please feel free to raise an issue at [https://github.com/mlcommons/mlperf-automations](https://github.com/mlcommons/mlperf-automations). diff --git a/mkdocs.yml b/mkdocs.yml index 634b4fd21f..175690dccb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -34,26 +34,17 @@ nav: - External Use: - SCC24 Guide: benchmarks/text_to_image/reproducibility/scc24.md - 2D Object Detection: - - RetinaNet: benchmarks/object_detection/retinanet.md - Yolo: benchmarks/object_detection/yolo.md - - Automotive: - - 3D Object Detection: - - PointPainting: benchmarks/automotive/3d_object_detection/pointpainting.md - Medical Imaging: - 3d-unet: benchmarks/medical_imaging/3d-unet.md - Language Processing: - Bert-Large: benchmarks/language/bert.md - - GPT-J: benchmarks/language/gpt-j.md - LLAMA2-70B: - Run Commands: benchmarks/language/llama2-70b.md - External Use: - SCC25 Guide: benchmarks/language/scc25_guide/scc25.md - - LLAMA3-405B: benchmarks/language/llama3_1-405b.md - LLAMA3-8B: benchmarks/language/llama3_1-8b.md - - MIXTRAL-8x7B: benchmarks/language/mixtral-8x7b.md - DeepSeek-R1: benchmarks/language/deepseek-r1.md - - Recommendation: - - DLRM-v2: benchmarks/recommendation/dlrm-v2.md - Graph Neural Networks: - R-GAT: benchmarks/graph/rgat.md - Speech to Text: @@ -65,6 +56,8 @@ nav: - Automated Submission CLI: submission/submission-cli.md - Power: - power/index.md + - System Info: + - system-info/index.md - Release Notes: - What's New: changelog/index.md - Changelog: changelog/changelog.md