perforce_rules.yml

groups:
- name: alert.rules
  rules:
  - alert: P4D service not running
    expr: node_systemd_unit_state{state="active",name=~"p4d_.*.service"} != 1
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: Endpoint {{ $labels.instance }} p4d service not running
      description: '{{ $labels.instance }} has been down for 2 mins.'
      runbook_id: _p4d_service_not_running
  - alert: P4prometheus service not running
    expr: node_systemd_unit_state{state="active",name="p4prometheus.service"} != 1
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: Endpoint {{ $labels.instance }} p4prometheus service not running
      description: '{{ $labels.instance }} has been down for 10 mins.'
      runbook_id: _p4prometheus_service_not_running
  - alert: P4Metrics service not running
    expr: node_systemd_unit_state{name="p4metrics.service", state!="active", instance!~"(?i).*proxy.*"} == 1
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: Endpoint {{ $labels.instance }} p4metrics service not running
      description: '{{ $labels.instance }} has been down for 10 mins.'
      runbook_id: _p4metrics_service_not_running
  - alert: Swarm unhealthy
    expr: p4_swarm_error != 0
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: Endpoint {{ $labels.instance }} swarm is unhealthy
      description: '{{ $labels.instance }} has been down for 10 mins.'
      runbook_id: _p4prometheus_swarm_unhealthy
  - alert: Node Exporter parsing issue
    expr: increase(node_textfile_scrape_error[10m]) > 0
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: Node exporter parsing issue on {{ $labels.instance }}
      description: Check the format of *.prom in /p4/metrics
      runbook_id: _node_exporter_parse_error_detected
  - alert: OOM Issue detected by p4metrics
    expr: increase(p4_memlimit_kill_candidates[10m]) > 0
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: OOM issue on {{ $labels.instance }}
      description: One or more processes were exceeding out-of-memory thresholds by p4metrics
      runbook_id: _oom_p4metrics_issue_detected
  - alert: OOM Kill made by p4metrics
    expr: increase(p4_memlimit_kills_total[10m]) > 0
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: OOM kill on {{ $labels.instance }}
      description: One or more processes were killed by p4metrics as over out-of-memory thresholds
      runbook_id: _oom_p4metrics_issue_detected
  - alert: OOM Kill Detected
    expr: increase(node_vmstat_oom_kill[10m]) > 0
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: OOM kill detected on {{ $labels.instance }}
      description: One or more processes were killed due to out-of-memory in the last 5 minutes.
      runbook_id: _oom_kill_detected
  - alert: P4D urgent license expiry
    expr: (p4_license_time_remaining{serverid!~".*ffr.*|.*edge.*|.*EDGE.*|.*[-_]ha.*"} / (24 * 60 * 60)) < 5
    for: 6h
    labels:
      severity: warning
    annotations:
      summary: Endpoint {{ $labels.instance }} license due to expire urgently (in {{ $value | printf "%.02f" }} days)
      description: '{{ $labels.instance }} has been low for 6 hours.'
      runbook_id: _p4d_urgent_license_expiry
  - alert: P4D license expiry
    expr: (p4_license_time_remaining{serverid!~".*ffr.*|.*edge.*|.*EDGE.*|.*[-_]ha.*"} / (24 * 60 * 60)) < 14
    for: 6h
    labels:
      severity: low
    annotations:
      summary: Endpoint {{ $labels.instance }} license due to expire (in {{ $value  | printf "%.02f" }} days)
      description: '{{ $labels.instance }} has been low for 6 hours.'
      runbook_id: _p4d_license_expiry
  - alert: Checkpoint Not Taken
    expr: ((time() - p4_sdp_checkpoint_log_time{serverid=~".*master.*|.*edge.*|.*EDGE.*",instance!~".*p4p.*|.*proxy.*|.*[-_]ha.*"}) / (60 * 60)) > 25
    for: 1h
    labels:
      severity: warning
    annotations:
      summary: Endpoint {{ $labels.instance }} checkpoint missing warning ({{ $value | printf "%.02f" }} hours)
      description: '{{ $labels.instance }} has been above target for 1 hour.'
      runbook_id: _checkpoint_not_taken
  - alert: P4D SSL certificate expiry
    expr: ((p4_ssl_cert_expires - time()) / (3600 * 24)) < 14
    for: 2h
    labels:
      severity: warning
    annotations:
      summary: Endpoint {{ $labels.instance }} P4D SSL Certificate expiry warning ({{ $value | printf "%.02f" }} days)
      description: '{{ $labels.instance }} has been above target for 2 hours.'
      runbook_id: _p4d_ssl_certificate_expiry
  - alert: HAS SSL certificate expiry
    expr: ((p4_auth_ssl_cert_expires - time()) / (3600 * 24)) < 14
    for: 2h
    labels:
      severity: warning
    annotations:
      summary: Endpoint {{ $labels.instance }} HAS (Helix Auth) SSL Certificate expiry warning ({{ $value | printf "%.02f" }} days)
      description: '{{ $labels.instance }} has been above target for 2 hours.'
      runbook_id: _p4d_has_ssl_certificate_expiry
  - alert: Replication Errors
    expr: p4_pull_replication_error > 0
    for: 30m
    labels:
      severity: high
    annotations:
      summary: Endpoint {{ $labels.instance }} replication error occurring
      description: '{{ $labels.instance }} has been true for 30m.'
      runbook_id: _replication_error
  - alert: Replication Slow
    expr: p4_pull_replica_lag / (1024 * 1024) > 100
    for: 2h
    labels:
      severity: warning
    annotations:
      summary: Endpoint {{ $labels.instance }} replication slow (metadata pull queue {{ $value | humanize }} behind }})
      description: '{{ $labels.instance }} has been true for 2h.'
      runbook_id: _replication_slow
  - alert: Diskspace Percentage Used Above Percentage Threshold
    expr: |
      (100.0 - 100 * min by (instance, mountpoint) (
        node_filesystem_avail_bytes{mountpoint=~"/hx.*"}
        / on (instance, mountpoint, device) node_filesystem_size_bytes{mountpoint=~"/hx.*"}
      )) > 90
    for: 2h
    labels:
      severity: warning
    annotations:
      summary: |
        Endpoint {{ $labels.instance }} for {{ $labels.mountpoint }} disk space percentage is {{ $value | printf "%.02f" }}% full ({{ printf "node_filesystem_avail_bytes{mountpoint='%s',instance='%s'}" $labels.mountpoint $labels.instance | query | first | value | humanize1024 }} free of {{ printf "node_filesystem_size_bytes{mountpoint='%s',instance='%s'}" $labels.mountpoint $labels.instance | query | first | value | humanize1024 }} total)
      description: '{{ $labels.instance }} has been true for 2 hours.'
      runbook_id: _diskspace_percentage_used_above_percentage_threshold
  - alert: Swarm Opt Diskspace High
    expr: |
      (100.0 - 100 * min by (customer, instance, mountpoint) (
        node_filesystem_avail_bytes{instance=~"(?i).*swarm.*",mountpoint="/opt"}
        / on (customer, instance, mountpoint, device) node_filesystem_size_bytes{instance=~"(?i).*swarm.*",mountpoint="/opt"}
      )) > 80
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: Endpoint {{ $labels.instance }} Swarm /opt disk space is {{ $value | printf "%.02f" }}% full
      description: Swarm data and logs are on /opt; investigate growth or expand the volume.
      runbook_id: _diskspace_percentage_used_above_percentage_threshold
  - alert: Diskspace Below Filesys Config /hxlogs - P4D STOPPED!
    expr: |
      min by (instance, mountpoint) (node_filesystem_avail_bytes{mountpoint="/hxlogs"}) -
            on (instance) p4_filesys_min{filesys="P4LOG"} < 0 or
      min by (instance, mountpoint) (node_filesystem_avail_bytes{mountpoint="/hxlogs"}) -
            on (instance) p4_filesys_min{filesys="P4JOURNAL"} < 0
    for: 3m
    labels:
      severity: high
    annotations:
      summary: Endpoint {{ $labels.instance }} disk space is {{$value | humanize}} below filesys.*.min NOW!!!
      description: '{{ $labels.instance }} has been true for 3 mins.'
      runbook_id: _diskspace_below_filesys_config_hxlogs_p4d_stopped
  - alert: Diskspace Below Filesys Config /hxmetadata - P4D STOPPED!
    expr: |
      min by (instance, mountpoint) (node_filesystem_avail_bytes{mountpoint="/hxmetadata"}) -
          on (instance) p4_filesys_min{filesys="P4ROOT"} < 0
    for: 3m
    labels:
      severity: high
    annotations:
      summary: Endpoint {{ $labels.instance }} disk space is {{$value | humanize}} below filesys.*.min NOW!!!
      description: '{{ $labels.instance }} has been true for 3 mins.'
      runbook_id: _diskspace_below_filesys_config_hxmetadata_p4d_stopped
  - alert: Diskspace Below Filesys Config /hxdepots - P4D STOPPED!
    expr: |
      min by (instance, mountpoint) (node_filesystem_avail_bytes{mountpoint="/hxdepots",instance!~".*proxy.*"}) -
          on (instance) p4_filesys_min{filesys="depot"} < 0
    for: 3m
    labels:
      severity: high
    annotations:
      summary: Endpoint {{ $labels.instance }} disk space is {{$value | humanize}} below filesys.*.min NOW!!!
      description: '{{ $labels.instance }} has been true for 3 mins.'
      runbook_id: _diskspace_below_filesys_config_hxdepots_p4d_stopped
  - alert: HX Diskspace Predicted Low /hxdepots
    expr: |
      min by (instance, mountpoint) (predict_linear(node_filesystem_avail_bytes{mountpoint="/hxdepots"}[1h], 1 * 24 * 3600)) -
         on (instance) p4_filesys_min{filesys="depot"} < 0
    for: 4h
    labels:
      severity: warning
    annotations:
      summary: '{{ $labels.instance }} for /hxdepots disk space predicting to go below filesys.*.min (by {{$value | humanize }}) in 24 hours'
      description: '{{ $labels.instance }} has been true for 4 hours.'
      runbook_id: _hx_diskspace_predicted_low
  - alert: HX Diskspace Predicted Low /hxlogs
    expr: |
      min by (instance, mountpoint) (predict_linear(node_filesystem_avail_bytes{mountpoint="/hxlogs"}[1h], 1 * 24 * 3600)) -
       on (instance) p4_filesys_min{filesys="P4LOG"} < 0 or
      min by (instance, mountpoint) (predict_linear(node_filesystem_avail_bytes{mountpoint="/hxlogs"}[1h], 1 * 24 * 3600)) -
       on (instance) p4_filesys_min{filesys="P4JOURNAL"} < 0
    for: 4h
    labels:
      severity: warning
    annotations:
      summary: '{{ $labels.instance }} for /hxlogs disk space predicting to go below filesys.*.min (by {{$value | humanize }}) in 24 hours'
      description: '{{ $labels.instance }} has been true for 4 hours.'
      runbook_id: _hx_diskspace_predicted_low
  - alert: HX Diskspace Predicted Low /hxmetadata
    expr: |
      min by (instance, mountpoint) (predict_linear(node_filesystem_avail_bytes{mountpoint="/hxmetadata"}[1h], 1 * 24 * 3600)) -
         on (instance) p4_filesys_min{filesys="P4ROOT"} < 0
    for: 4h
    labels:
      severity: warning
    annotations:
      summary: '{{ $labels.instance }} for /hxmetadata disk space predicting to go below filesys.*.min (by {{$value | humanize }}) in 24 hours'
      description: '{{ $labels.instance }} has been true for 4 hours.'
      runbook_id: _hx_diskspace_predicted_low
  - alert: Root Diskspace Predicted Low
    expr: |
      predict_linear(node_filesystem_avail_bytes{mountpoint="/"}[1h], 1 * 24 * 3600)  < 0
    for: 2h
    labels:
      severity: warning
    annotations:
      summary: Endpoint {{ $labels.instance }} for root disk space predicting to go below 0 (by {{$value | humanize }}) in 24 hours
      description: '{{ $labels.instance }} has been true for 2 hours.'
      runbook_id: _root_diskspace_predicted_low
  - alert: CPU Usage High
    expr: |
      (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"} [5m]) * 100))) > 80.0
    for: 90m
    labels:
      severity: warning
    annotations:
      summary: Endpoint {{ $labels.instance }} CPU usage above 80% (actual {{$value | printf "%.02f"}})
      description: '{{ $labels.instance }} has been true for 90 mins.'
      runbook_id: _cpu_usage_high
  - alert: App Memory Usage High
    expr: |
      (100 * (
          node_memory_MemTotal_bytes -
          node_memory_MemFree_bytes -
          node_memory_Buffers_bytes -
          node_memory_Cached_bytes -
          node_memory_SwapCached_bytes -
          node_memory_Slab_bytes -
          node_memory_PageTables_bytes -
          node_memory_VmallocUsed_bytes)
          / node_memory_MemTotal_bytes) > 70.0
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: Endpoint {{ $labels.instance }} for App Memory usage above 70% (actual {{$value | printf "%.02f"}})
      description: '{{ $labels.instance }} has been true for 10 mins.'
      runbook_id: _app_memory_usage_high
  - alert: Hansoft Diskspace Low
    expr: |
      predict_linear(node_filesystem_avail_bytes{instance=~".*hansoft.*",mountpoint="/"}[1h], 7 * 24 * 3600) < 0 or predict_linear(node_filesystem_avail_bytes{instance=~".*hansoft.*",mountpoint="/opt/HansoftServer"}[1h], 7 * 24 * 3600) < 0 or node_filesystem_avail_bytes{instance=~".*hansoft.*",mountpoint="/"} / (1024 * 1024 * 1024) < 5 or node_filesystem_avail_bytes{instance=~".*hansoft.*",mountpoint="/opt/HansoftServer"} / (1024 * 1024 * 1024) < 15
    for: 4h
    labels:
      severity: warning
    annotations:
      summary: Endpoint {{ $labels.instance }} for {{ $labels.mountpoint }} Hansoft disk space predicting to go below threshold in 7 days or is below threshold (actual {{$value | printf "%.02f"}})
      description: '{{ $labels.instance }} has been true for 4 hours.'
      runbook_id: _hansoft_diskspace_low