Implementation of custom bytetrack demo for real time stream analysis [GSoC'2026]

ci/lib_search.py

@@ -113,6 +113,7 @@ def check_dir(start_dir):
         'ovms-c/dist',
         'requirements.txt',
         'requirements_win.txt',
+        'bytetrack_ovms.pbtxt',
         'resnet_images.txt',
         "resnet_labels.txt",
         'rest_sdk_v2.10.16.patch',

demos/mediapipe/bytetrack/README.md

@@ -0,0 +1,70 @@
+# ByteTrack Demo Setup
+
+End-to-end demo: video source (webcam / file) → OpenVINO Model Server (YOLOX Tiny + ByteTrack) → output (screen / file).
+
+---
+
+## Steps
+
+### 1. Clone the repository
+
+Clone the repository, switch to the `gsoc_bytetrack` branch, and move into the demo directory:
+
+```bash
+git clone https://github.com/Vishwa2684/model_server
+cd model_server
+git checkout gsoc_bytetrack
+cd demos/mediapipe/bytetrack
+```
+
+### 2. Install requirements
+
+Install all the Python dependencies needed by the client and the model download script. Run this from inside the `demos/mediapipe/bytetrack` directory:
+
+```bash
+pip install -r requirements.txt
+```
+
+### 3. Download a model
+
+Download the detector model that the OpenVINO Model Server will use. This same command also fetches the COCO class list used for labeling detections:
+
+```bash
+python download_models.py --model OpenVINO/yolox_tiny-fp16-ov
+```
+
+> Swap `--model` for any of the repo IDs listed below to use a different YOLOX size.
+
+| Model | HuggingFace Repo |
+|---|---|
+| YOLOX-Tiny (fp16 precision)| `OpenVINO/yolox_tiny-fp16-ov` |
+| YOLOX-Tiny (int8 precision)| `OpenVINO/yolox_tiny-int8-ov` |
+
+`yolox_tiny-fp16-ov` is the default used in this demo.
+
+This step populates the local model directory that `config.json` (used by the OpenVINO Model Server in step 4) points to, and that ByteTrack consumes downstream for tracking.
+
+### 4. Start the OpenVINO Model Server
+
+Bring up the OpenVINO Model Server as a Docker container. This mounts your current directory into the container so it can read `config.json`, and exposes port 9000 for the client to connect to:
+
+```bash
+docker run -d -v $PWD:/demo -p 9000:9000 openvino/model_server:latest --config_path /demo/config.json --port 9000
+```
+
+Leave this container running in the background — the client in the next step connects to it over gRPC.
+
+### 5. Run the demo — local webcam → screen
+
+With the model server running, run the client script. This reads directly from your local webcam, runs it through detection + ByteTrack tracking, and renders the annotated output live in a window on your screen:
+
+```bash
+cd ../../real_time_stream_analysis/python
+python client.py --grpc_address localhost:9000 --input_stream 0 --output_stream screen --model_name ByteTrack --input_name input_video
+```
+
+- `--grpc_address localhost:9000` — address of the OpenVINO Model Server started in step 4.
+- `--input_stream 0` — camera device ID `0` (use `1`, `2`, etc. if you have multiple cameras and want a different one).
+- `--output_stream screen` — opens a live preview window instead of writing to a file or stream.
+
+A window should open showing your webcam feed with tracked bounding boxes drawn on it in real time. To use different input and output streams for real time. Read the documentation on [real time stream analysis](../../real_time_stream_analysis/python/README.md)

demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt

@@ -0,0 +1,138 @@
+input_stream: "IMAGE:input_video"
+output_stream: "IMAGE:output"
+
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE:input_video"
+  output_stream: "IMAGE:transformed_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
+      output_width: 416
+      output_height: 416
+    }
+  }
+}
+
+node {
+  calculator: "OpenVINOConverterCalculator"
+  input_stream: "IMAGE:transformed_input_video"
+  output_stream: "TENSORS:image_tensor"
+  node_options: {
+    [type.googleapis.com/mediapipe.OpenVINOConverterCalculatorOptions] {
+      enable_normalization: true
+      use_custom_normalization: true
+      custom_div: 1.0
+      custom_sub: 0.0
+    }
+  }
+}
+
+node {
+  calculator: "OpenVINOModelServerSessionCalculator"
+  output_side_packet: "SESSION:session"
+  node_options: {
+    [type.googleapis.com/mediapipe.OpenVINOModelServerSessionCalculatorOptions]: {
+      servable_name: "yolox_tiny-fp16-ov"  # servable name inside OVMS
+      servable_version: "1"
+    }
+  }
+}
+
+node {
+  calculator: "OpenVINOInferenceCalculator"
+  input_side_packet: "SESSION:session"
+  input_stream: "OVTENSORS:image_tensor"
+  output_stream: "OVTENSORS2:detection_tensors"
+  node_options: {
+    [type.googleapis.com/mediapipe.OpenVINOInferenceCalculatorOptions]: {
+          input_order_list :["image"]
+          output_order_list :["boxes","labels"]
+        }
+  }
+}
+
+### OpenVINOYoloXTensorsToDetectionsCalculator was developed for TFLite specific model, but we can use OVYoloXTensorsToDetectionsCalculator for OpenVINO model.
+
+#node{
+#    calculator: "OpenVINOYoloXTensorsToDetectionsCalculator"
+#    input_stream: "TENSORS:detection_tensors"
+#    output_stream: "DETECTIONS:detections"
+#    node_options: {
+#    [type.googleapis.com/mediapipe.OpenVINOYoloXTensorsToDetectionsCalculatorOptions] {
+#      conf_thresh: 0.1
+#    }
+#  }
+# }
+
+node {
+  calculator: "OVYoloXTensorsToDetectionsCalculator"
+  input_stream: "TENSORS:detection_tensors"
+  output_stream: "DETECTIONS:detections"
+
+  node_options: {
+    [type.googleapis.com/mediapipe.OVYoloXTensorsToDetectionsCalculatorOptions] {
+      conf_thresh: 0.1
+      input_size: 416.0
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "detections"
+  output_stream: "filtered_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
+      min_suppression_threshold: 0.45
+      max_num_detections: 100
+      overlap_type: INTERSECTION_OVER_UNION
+      return_empty_detections: true
+    }
+  }
+}
+
+
+# Maps detection label IDs to the corresponding label text. The label map is
+# provided in the label_map_path option.
+node {
+  calculator: "DetectionLabelIdToTextCalculator"
+  input_stream: "filtered_detections"
+  output_stream: "output_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
+      label_map_path: "/demo/coco_80cl.txt"
+    }
+  }
+}
+
+node {
+  calculator: "ByteTrackCalculator"
+  input_stream: "DETECTIONS:output_detections"
+  output_stream: "DETECTIONS:tracked_detections"
+  options: {
+    [mediapipe.ByteTrackCalculatorOptions.ext] {
+      track_high_threshold:0.7
+      track_low_threshold:0.55
+      new_track_threshold:0.35
+      matching_threshold: 0.8
+      track_buffer: 60
+      fuse_score: false
+    }
+  }
+}
+
+# Converts the detections to drawing primitives for annotation overlay.
+node {
+  calculator: "DetectionColorByIdCalculator"
+  input_stream: "DETECTIONS:tracked_detections"
+  output_stream: "RENDER_DATA:detections_render_data"
+}
+
+# Draws annotations and overlays them on top of the input images.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "IMAGE:input_video"
+  input_stream: "detections_render_data"
+  output_stream: "IMAGE:output"
+}

demos/mediapipe/bytetrack/config.json

@@ -0,0 +1,16 @@
+{
+    "model_config_list": [
+        {"config": {
+            "name": "yolox_tiny-fp16-ov",
+            "base_path": "yolox_tiny-fp16-ov"
+    }
+    }
+    ],
+    "mediapipe_config_list": [
+    {
+        "name":"ByteTrack",
+        "base_path":"./",
+        "graph_path":"bytetrack_ovms.pbtxt"
+    }
+    ]
+}

demos/mediapipe/bytetrack/download_models.py

@@ -0,0 +1,120 @@
+#
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import glob
+import json
+import argparse
+import os
+import openvino as ov
+from huggingface_hub import snapshot_download
+
+os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1"
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--model-repo",
+    default="OpenVINO/yolox_tiny-fp16-ov",
+    help="Hugging Face model repository",
+)
+args = parser.parse_args()
+
+MODEL_REPO = args.model_repo
+MODEL_NAME = MODEL_REPO.split("/")[-1]
+# ---------------------------------------------------------
+# 1. Download model
+# ---------------------------------------------------------
+
+model_dir = snapshot_download(repo_id=MODEL_REPO)
+
+xml_path = glob.glob(os.path.join(model_dir, "*.xml"))[0]
+config_path = os.path.join(model_dir, "config.json")
+
+print("Found IR    :", xml_path)
+print("Found config:", config_path)
+
+
+# ---------------------------------------------------------
+# 2. Read config.json
+# ---------------------------------------------------------
+
+with open(config_path, "r") as f:
+    config = json.load(f)
+
+print("\nModel config:")
+print("model_name :", config.get("model_name"))
+print("model_type :", config.get("model_type"))
+print("input_type :", config.get("input_dtype"))
+print("mean_values:", config.get("mean_values"))
+print("scale_values:", config.get("scale_values"))
+print("classes:", config.get("labels"))
+
+# ---------------------------------------------------------
+# 3. Prepare classes list
+# ---------------------------------------------------------
+classes = config.get("labels").split(" ")
+
+# ---------------------------------------------------------
+# 4. Parse mean and scale values
+# ---------------------------------------------------------
+mean_values = [float(x) for x in config["mean_values"].split()]
+scale_values = [float(x) for x in config["scale_values"].split()]
+
+print("\nParsed preprocessing:")
+print("mean :", mean_values)
+print("scale:", scale_values)
+
+# ---------------------------------------------------------
+# 5. Load OpenVINO model
+# ---------------------------------------------------------
+core = ov.Core()
+model = core.read_model(xml_path)
+# ---------------------------------------------------------
+# 6. Configure preprocessing
+# ---------------------------------------------------------
+ppp = ov.preprocess.PrePostProcessor(model)
+inp = ppp.input(0)
+
+# Input coming from user/image:
+# f32 NHWC
+inp.tensor().set_element_type(ov.Type.f32).set_layout(ov.Layout("NHWC"))
+
+# Model expects:
+# float32 NCHW
+inp.model().set_layout(ov.Layout("NCHW"))
+
+# Preprocessing:
+
+inp.preprocess().convert_element_type(ov.Type.f32).convert_layout(
+    ov.Layout("NCHW")
+).scale(255.0).mean(mean_values).scale(scale_values)
+# ---------------------------------------------------------
+# 7. Build and save
+# ---------------------------------------------------------
+
+model = ppp.build()
+
+output_path = f"{MODEL_NAME}/1/{MODEL_NAME}.xml"
+
+ov.save_model(model, output_path)
+
+print("\nSaved:", os.path.abspath(output_path))
+
+with open("coco_80cl.txt", "w") as f:
+    n = len(classes)
+    for i, c in enumerate(classes):
+        f.write(c + ("\n" if i < n - 1 else ""))
+
+print("Downloaded successfully")

demos/real_time_stream_analysis/python/client.py

@@ -23,6 +23,8 @@
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--grpc_address', required=False, default='localhost:9022', help='Specify url to grpc service')
+parser.add_argument('--ffmpeg_output_width', required=False, default=None, type=int, help='Width of the output video')
+parser.add_argument('--ffmpeg_output_height', required=False, default=None, type=int, help='Height of the output video')
 parser.add_argument('--input_stream', required=False, default="rtsp://localhost:8080/channel1", type=str, help='Url of input rtsp stream')
 parser.add_argument('--output_stream', required=False, default="rtsp://localhost:8080/channel2", type=str, help='Url of output rtsp stream')
 parser.add_argument('--model_name', required=False, default="holisticTracking", type=str, help='Name of the model')
@@ -54,6 +56,6 @@ def postprocess(frame, result):
     backend = StreamClient.OutputBackends.cv2
     exact = True
 
-client = StreamClient(postprocess_callback = postprocess, preprocess_callback=preprocess, output_backend=backend, source=args.input_stream, sink=args.output_stream, exact=exact, benchmark=args.benchmark, verbose=args.verbose)
+client = StreamClient(postprocess_callback = postprocess, preprocess_callback=preprocess, output_backend=backend, source=args.input_stream, sink=args.output_stream, exact=exact, benchmark=args.benchmark, verbose=args.verbose, ffmpeg_output_width=args.ffmpeg_output_width, ffmpeg_output_height=args.ffmpeg_output_height)
 client.start(ovms_address=args.grpc_address, input_name=args.input_name, model_name=args.model_name, datatype = StreamClient.Datatypes.uint8, batch = False, limit_stream_duration = args.limit_stream_duration, limit_frames = args.limit_frames, streaming_api=True)
 

spelling-whitelist.txt

@@ -39,5 +39,8 @@ windows_parse_tests.bat:141: SEH ==> SHE
 windows_parse_tests.bat:144: SEH ==> SHE
 src/test/llm/output_parsers/gemma4_output_parser_test.cpp
 src/test/llm/output_parsers/qwen3_output_parser_test.cpp:719: thi ==> the, this
+STrack
+strack
+nd
 extras/chat_template_examples/chat_template_onyx.jinja
 src/test/llm/chat_templates/chat_template_onyx.jinja

src/BUILD

@@ -768,6 +768,8 @@ ovms_cc_library(
                 "//src/image_gen:image_gen_calculator",
                 "//src/audio/speech_to_text:s2t_calculator",
                 "//src/audio/text_to_speech:t2s_calculator",
+                "//src/yolox:ov_yolox_tensors_to_detections_calculator",
+                "//src/bytetrack/calculators:bytetrack_calculators",
                 "//src/audio:audio_utils",
                 "//src/image_gen:imagegen_init",
                 "//src/llm:openai_responses_handler",