Implementation of custom bytetrack demo for real time stream analysis [GSoC'2026]

WORKSPACE

@@ -176,8 +176,9 @@ http_archive(
 #### Will be used on feature release
 git_repository(
     name = "mediapipe",
-    remote = "https://github.com/openvinotoolkit/mediapipe",
-    commit = "12e8d511cfbc5f471c498278a65a02dd250963e8", # top of mediapipe main branch as of 26.11.2025
+    remote = "https://github.com/Vishwa2684/mediapipe_ovms",
+    # commit = "12e8d511cfbc5f471c498278a65a02dd250963e8", # top of mediapipe main branch as of 26.11.2025
+    branch = "custom_bytetrack_graph"
 )
 
 # DEV mediapipe 1 source - adjust local repository path for build

demos/mediapipe/bytetrack/README.md

@@ -0,0 +1,107 @@
+# ByteTrack Demo Setup
+
+## 1. Download the YOLOX Tiny ONNX Model
+
+Download the YOLOX Tiny ONNX model from the official release:
+
+https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/yolox_tiny.onnx
+
+---
+
+## 2. Convert the ONNX Model to TensorFlow Lite
+
+Open a Google Colab notebook and:
+
+1. Install `onnx2tf`.
+2. Upload `yolox_tiny.onnx` to the notebook.
+3. Run:
+
+```bash
+!onnx2tf -i yolox_tiny.onnx -o yolox_tiny
+```
+
+This generates the TensorFlow Lite model.
+
+---
+
+## 3. Download COCO Class Labels
+
+Download the COCO 80-class label file:
+
+https://raw.githubusercontent.com/openvinotoolkit/open_model_zoo/master/data/dataset_classes/coco_80cl.txt
+
+---
+
+# Running the Demo
+
+## 1. Start the OpenVINO Model Server
+
+```bash
+docker run -d \
+    -v $PWD:/demo \
+    -p 9000:9000 \
+    openvino/model_server:latest \
+    --config_path /demo/config.json \
+    --port 9000
+```
+
+---
+
+## 2. Create an RTSP Input Stream
+
+Use FFmpeg to publish your webcam as an RTSP stream.
+
+```bash
+ffmpeg -f dshow -video_size 1280x720 \
+-i video="HP True Vision FHD Camera" \
+-f rtsp -rtsp_transport tcp \
+rtsp://localhost:8554/channel1
+```
+
+> **Work in Progress:** The following H.264-based streaming command is still being evaluated and may not work correctly in all setups.
+
+```bash
+ffmpeg -f dshow \
+-video_size 1280x720 \
+-framerate 30 \
+-i video="HP True Vision FHD Camera" \
+-c:v libx264 \
+-crf 18 \
+-preset veryfast \
+-f rtsp \
+-rtsp_transport tcp \
+rtsp://localhost:8554/channel1
+```
+
+---
+
+## 3. Run the Real-Time Stream Analysis Client
+
+```bash
+python client.py \
+    --grpc_address localhost:9000 \
+    --input_stream rtsp://localhost:8554/channel1 \
+    --output_stream rtsp://localhost:8554/channel2 \
+    --model_name ByteTrack \
+    --input_name input_video
+```
+
+---
+
+## 4. View the Output Stream
+
+**Option 1 (recommended):**
+
+```bash
+ffplay -rtsp_transport tcp \
+-vf "scale=704:704,format=yuv420p" \
+rtsp://localhost:8554/channel2
+```
+
+**Option 2 (verbose logging):**
+
+```bash
+ffplay -loglevel verbose \
+-rtsp_transport tcp \
+rtsp://localhost:8554/channel2
+```

demos/mediapipe/bytetrack/bytetrack_ovms.pbtxt

@@ -0,0 +1,127 @@
+input_stream: "IMAGE:input_video"
+output_stream: "IMAGE:output"
+
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE:input_video"
+  output_stream: "IMAGE:transformed_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
+      output_width: 416
+      output_height: 416
+    }
+  }
+}
+
+node {
+  calculator: "OpenVINOConverterCalculator"
+  input_stream: "IMAGE:transformed_input_video"
+  output_stream: "TENSORS:image_tensor"
+  node_options: {
+    [type.googleapis.com/mediapipe.OpenVINOConverterCalculatorOptions] {
+      enable_normalization: true
+      use_custom_normalization: true
+      custom_div: 1.0
+      custom_sub: 0.0
+    }
+  }
+}
+# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "OpenVINOModelServerSessionCalculator"
+  output_side_packet: "SESSION:session"
+  node_options: {
+    [type.googleapis.com/mediapipe.OpenVINOModelServerSessionCalculatorOptions]: {
+      servable_name: "yoloxt_float32"  # servable name inside OVMS
+      servable_version: "1"
+    }
+  }
+}
+
+node {
+  calculator: "OpenVINOInferenceCalculator"
+  input_side_packet: "SESSION:session"
+  input_stream: "OVTENSORS:image_tensor"
+  output_stream: "OVTENSORS2:detection_tensors"
+  node_options: {
+    [type.googleapis.com/mediapipe.OpenVINOInferenceCalculatorOptions]: {
+          input_order_list :["images"]
+          output_order_list :["output"]
+        }
+  }
+}
+
+### WRITE YOLO SPECIFIC CALCULATORS
+
+node{
+    calculator: "OpenVINOYoloXTensorsToDetectionsCalculator"
+    input_stream: "TENSORS:detection_tensors"
+    output_stream: "DETECTIONS:detections"
+    node_options: {
+    [type.googleapis.com/mediapipe.OpenVINOYoloXTensorsToDetectionsCalculatorOptions] {
+      conf_thresh: 0.1
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "detections"
+  output_stream: "filtered_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
+      min_suppression_threshold: 0.45
+      max_num_detections: 100
+      overlap_type: INTERSECTION_OVER_UNION
+      return_empty_detections: true
+    }
+  }
+}
+
+
+# Maps detection label IDs to the corresponding label text. The label map is
+# provided in the label_map_path option.
+node {
+  calculator: "DetectionLabelIdToTextCalculator"
+  input_stream: "filtered_detections"
+  output_stream: "output_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
+      label_map_path: "/demo/coco_80cl.txt"
+    }
+  }
+}
+
+node {
+  calculator: "ByteTrackCalculator"
+  input_stream: "DETECTIONS:output_detections"
+  output_stream: "DETECTIONS:tracked_detections"
+  options: {
+    [mediapipe.ByteTrackCalculatorOptions.ext] {
+      track_high_threshold:0.7
+      track_low_threshold:0.55
+      new_track_threshold:0.35
+      matching_threshold: 0.8
+      track_buffer: 60
+      fuse_score: false
+    }
+  }
+}
+
+# Converts the detections to drawing primitives for annotation overlay.
+node {
+  calculator: "DetectionColorByIdCalculator"
+  input_stream: "DETECTIONS:tracked_detections"
+  output_stream: "RENDER_DATA:detections_render_data"
+}
+
+# Draws annotations and overlays them on top of the input images.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "IMAGE:input_video"
+  input_stream: "detections_render_data"
+  output_stream: "IMAGE:output"
+}

demos/mediapipe/bytetrack/config.json

@@ -0,0 +1,16 @@
+{
+    "model_config_list": [
+        {"config": {
+            "name": "yoloxt_float32",
+            "base_path": "yolox_tiny_float32"
+    }
+    }
+    ],
+    "mediapipe_config_list": [
+    {
+        "name":"ByteTrack",
+        "base_path":"./",
+        "graph_path":"bytetrack_ovms.pbtxt"
+    }
+    ]
+}

demos/real_time_stream_analysis/python/client.py

@@ -23,6 +23,8 @@
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--grpc_address', required=False, default='localhost:9022', help='Specify url to grpc service')
+parser.add_argument('--ffmpeg_output_width', required=False, default=None, type=int, help='Width of the output video')
+parser.add_argument('--ffmpeg_output_height', required=False, default=None, type=int, help='Height of the output video')
 parser.add_argument('--input_stream', required=False, default="rtsp://localhost:8080/channel1", type=str, help='Url of input rtsp stream')
 parser.add_argument('--output_stream', required=False, default="rtsp://localhost:8080/channel2", type=str, help='Url of output rtsp stream')
 parser.add_argument('--model_name', required=False, default="holisticTracking", type=str, help='Name of the model')
@@ -54,6 +56,6 @@ def postprocess(frame, result):
     backend = StreamClient.OutputBackends.cv2
     exact = True
 
-client = StreamClient(postprocess_callback = postprocess, preprocess_callback=preprocess, output_backend=backend, source=args.input_stream, sink=args.output_stream, exact=exact, benchmark=args.benchmark, verbose=args.verbose)
+client = StreamClient(postprocess_callback = postprocess, preprocess_callback=preprocess, output_backend=backend, source=args.input_stream, sink=args.output_stream, exact=exact, benchmark=args.benchmark, verbose=args.verbose,ffmpeg_output_width=args.ffmpeg_output_width, ffmpeg_output_height=args.ffmpeg_output_height)
 client.start(ovms_address=args.grpc_address, input_name=args.input_name, model_name=args.model_name, datatype = StreamClient.Datatypes.uint8, batch = False, limit_stream_duration = args.limit_stream_duration, limit_frames = args.limit_frames, streaming_api=True)
 

src/kfs_frontend/kfs_graph_executor_impl.cpp

@@ -1027,15 +1027,16 @@ static inline Status checkTimestamp(const KFSRequest& request, const ::mediapipe
     return StatusCode::OK;
 }
 
+// this method
 static Status deserializeTimestampIfAvailable(
     const KFSRequest& request,
-    ::mediapipe::Timestamp& timestamp) {
+    ::mediapipe::Timestamp& timestamp,
+    const ::mediapipe::Timestamp& lastPushedTimestamp) {  // pass in current stream state
     auto timestampParamIt = request.parameters().find(TIMESTAMP_PARAMETER_NAME);
     if (timestampParamIt != request.parameters().end()) {
         SPDLOG_DEBUG("Found {} timestamp parameter in request for: {}", TIMESTAMP_PARAMETER_NAME, request.model_name());
         auto& parameterChoice = timestampParamIt->second;
         if (parameterChoice.parameter_choice_case() == inference::InferParameter::ParameterChoiceCase::kInt64Param) {
-            // Cannot create with error checking since error check = abseil death test
             timestamp = ::mediapipe::Timestamp::CreateNoErrorChecking(parameterChoice.int64_param());
             if (!timestamp.IsRangeValue()) {
                 SPDLOG_DEBUG("Timestamp not in range: {}; for request to: {};", timestamp.DebugString(), request.model_name());
@@ -1047,8 +1048,17 @@ static Status deserializeTimestampIfAvailable(
             return status;
         }
     } else {
-        auto now = std::chrono::system_clock::now();
-        timestamp = ::mediapipe::Timestamp(std::chrono::duration_cast<std::chrono::microseconds>(now.time_since_epoch()).count());
+        // steady_clock never jumps backward; system_clock can
+        auto now = std::chrono::steady_clock::now().time_since_epoch();
+        int64_t candidate = std::chrono::duration_cast<std::chrono::microseconds>(now).count();
+        timestamp = ::mediapipe::Timestamp::CreateNoErrorChecking(candidate);
+    }
+
+    // Enforce strict monotonicity regardless of source
+    if (lastPushedTimestamp != ::mediapipe::Timestamp::Unset() && timestamp <= lastPushedTimestamp) {
+        SPDLOG_DEBUG("Non-monotonic timestamp detected: new={} <= last={}; for request to: {}",
+            timestamp.DebugString(), lastPushedTimestamp.DebugString(), request.model_name());
+        return Status(StatusCode::MEDIAPIPE_INVALID_TIMESTAMP, "Timestamp did not increase relative to previous packet in stream");
     }
     return StatusCode::OK;
 }
@@ -1133,6 +1143,7 @@ Status onPacketReadySerializeImpl(
     return status;
 }
 
+// this method
 Status createAndPushPacketsImpl(
     std::shared_ptr<const KFSRequest> request,
     stream_types_mapping_t& inputTypes,
@@ -1141,7 +1152,8 @@ Status createAndPushPacketsImpl(
     ::mediapipe::Timestamp& currentTimestamp,
     size_t& numberOfPacketsCreated) {
 
-    OVMS_RETURN_ON_FAIL(deserializeTimestampIfAvailable(*request, currentTimestamp));
+    const ::mediapipe::Timestamp lastPushedTimestamp = currentTimestamp;  // snapshot BEFORE overwrite
+    OVMS_RETURN_ON_FAIL(deserializeTimestampIfAvailable(*request, currentTimestamp, lastPushedTimestamp));
     OVMS_RETURN_ON_FAIL(checkTimestamp(*request, currentTimestamp));
     OVMS_RETURN_ON_FAIL(validateRequestCoherencyKFS(*request, request->model_name(), MediapipeGraphDefinition::VERSION));
 

third_party/mediapipe_calculators/BUILD

@@ -108,6 +108,8 @@ cc_library(
         "@mediapipe//mediapipe/calculators/video:box_detector_calculator",
         "@mediapipe//mediapipe/calculators/video:tracked_detection_manager_calculator",
         "@mediapipe//mediapipe/calculators/video:video_pre_stream_calculator",
+        #BYTETRACK CALCULATORS (GSoC'2026 - Vishwa2684)
+        "@mediapipe//mediapipe/graphs/bytetrack/calculators:bytetrack_calculators",
     ]  + select({
         "//conditions:default": [
             "@mediapipe//mediapipe/calculators/core:packet_cloner_calculator", # TODO windows: stdc++20 required