diff --git a/cilium/tests/test_unit.py b/cilium/tests/test_unit.py index 8a6f609c078d2..4ba777ad6724b 100644 --- a/cilium/tests/test_unit.py +++ b/cilium/tests/test_unit.py @@ -91,6 +91,30 @@ def test_v2_both_endpoints_build_two_scrapers(): assert endpoints == ["http://agent/metrics", "http://op/metrics"] +def test_v2_metric_limit_issue_uses_generated_scraper_endpoints(datadog_agent): + agent_endpoint = "http://z-agent/metrics" + operator_endpoint = "http://a-operator/metrics" + check = CiliumCheckV2( + "cilium", + {}, + [{"agent_endpoint": agent_endpoint, "operator_endpoint": operator_endpoint}], + ) + check._parse_config() + check.configure_scrapers() + + check._on_metric_limit_state(True, 20, 5) + + [issue] = datadog_agent._sent_reported_issues["cilium"] + assert "openmetrics_endpoint" not in check.instance + assert issue["id"] == "openmetrics-dropped-config:659a0d48b0fa9caf" + assert issue["title"] == "Dropping 15 of 20 OpenMetrics metrics" + assert issue["extra"]["endpoints"] == [operator_endpoint, agent_endpoint] + assert "2 configured endpoints" in issue["description"] + assert "totals cover the complete check run" in issue["description"] + assert agent_endpoint in issue["description"] + assert operator_endpoint in issue["description"] + + def test_v2_agent_scraper_uses_agent_metrics_not_operator(): check = CiliumCheckV2("cilium", {}, [{"agent_endpoint": "http://agent/metrics"}]) check._parse_config() diff --git a/datadog_checks_base/changelog.d/24819.added b/datadog_checks_base/changelog.d/24819.added new file mode 100644 index 0000000000000..bb602d28c69f8 --- /dev/null +++ b/datadog_checks_base/changelog.d/24819.added @@ -0,0 +1 @@ +Add an Agent Health issue when OpenMetrics checks drop metrics at the configured limit. diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index 6105348ad9857..e0f051a7f524a 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -1661,6 +1661,15 @@ def run(self): else: self.check(instance) + if self.metric_limiter: + try: + reached_limit = self.metric_limiter.reached_limit + observed_count = self.metric_limiter.count + limit = self.metric_limiter.limit + self._on_metric_limit_state(reached_limit, observed_count, limit) + except Exception: + self.log.debug('Error handling metric limit state', exc_info=True) + error_report = '' except Exception as e: message = self.sanitize(str(e)) @@ -1682,6 +1691,10 @@ def run(self): return error_report + def _on_metric_limit_state(self, reached_limit: bool, observed_count: int, limit: int) -> None: + """Called once per run for checks with an active metric limiter.""" + pass + def run_check_initializations(self): while self.check_initializations: initialization = self.check_initializations.popleft() @@ -1877,6 +1890,8 @@ def load_config(yaml_str: str) -> Any: # Remediation *Remediation `protobuf:"bytes,11,opt,name=remediation,proto3" json:"remediation,omitempty"` # // Tags are additional labels for the issue # Tags []string `protobuf:"bytes,12,rep,name=tags,proto3" json:"tags,omitempty"` + # // IssueType snake_case version of issue name + # IssueType string `protobuf:"bytes,14,opt,name=issue_type,json=issueType,proto3" json:"issue_type,omitempty"` # Remediation should be a dict with the following keys: # - summary: str @@ -1891,6 +1906,7 @@ def report_issue( self, id: str, issue_name: str, + issue_type: str, title: str = None, description: str = None, category: str = None, @@ -1899,14 +1915,17 @@ def report_issue( remediation: dict = None, tags: list = None, ): - # Issue ID and Name are required + # Issue ID, Name, and Type are required if not id: raise ValueError("Issue ID is required") if not issue_name: raise ValueError("Issue Name is required") + if not issue_type: + raise ValueError("Issue Type is required") issue = { 'id': id, 'issue_name': issue_name, + 'issue_type': issue_type, 'title': title, 'description': description, 'category': category, diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/base_check.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/base_check.py index 3a65a8dc83efb..99fd7e674d3b0 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/base_check.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/base_check.py @@ -9,6 +9,7 @@ from datadog_checks.base.errors import CheckException from datadog_checks.base.utils.tracing import traced_class +from .metric_limit_issue import MetricLimitIssueReporter from .mixins import OpenMetricsScraperMixin STANDARD_FIELDS = [ @@ -88,6 +89,9 @@ def __init__(self, *args, **kwargs): default_namespace = legacy_kwargs_in_args[1] super(OpenMetricsBaseCheck, self).__init__(*args, **kwargs) + self.metric_limit_issue_reporter: MetricLimitIssueReporter = MetricLimitIssueReporter( + filter_option_text='metrics / ignore_metrics' + ) self.config_map = {} self._http_handlers = {} self.default_instances = default_instances @@ -141,6 +145,15 @@ def check(self, instance): self.process(scraper_config) + def _on_metric_limit_state(self, reached_limit: bool, observed_count: int, limit: int) -> None: + self.metric_limit_issue_reporter.handle( + self, + (self.instance.get('prometheus_url'),), + reached_limit, + observed_count, + limit, + ) + def get_scraper_config(self, instance): """ Validates the instance configuration and creates a scraper configuration for a new instance. diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/metric_limit_issue.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/metric_limit_issue.py new file mode 100644 index 0000000000000..0d9b7710b13b9 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/metric_limit_issue.py @@ -0,0 +1,141 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + + from datadog_checks.base.checks import AgentCheck + +SEVERITY_HIGH_RATIO = 0.25 +SEVERITY_MEDIUM_RATIO = 0.05 + +ISSUE_NAME = 'OpenMetrics Metrics Dropped By Configured Limit' +ISSUE_TYPE = 'openmetrics_metrics_dropped_by_configured_limit' + + +@dataclass +class MetricLimitIssueReporter: + filter_option_text: str + + def handle( + self, + check: AgentCheck, + endpoints: Iterable[str | None] | None, + reached_limit: bool, + observed_count: int, + limit: int, + ) -> None: + """Report or resolve one configured-limit issue for an OpenMetrics check run.""" + endpoints = _normalize_endpoints(endpoints or ()) + if not endpoints: + check.log.debug('Cannot handle the OpenMetrics metric limit state without an endpoint') + return + + issue_id = _issue_id(check.hostname, check.name, endpoints, check.instance.get('namespace', '')) + + if reached_limit: + dropped = max(0, observed_count - limit) + ratio = dropped / observed_count + title, description = _issue_text(check.name, endpoints, limit, observed_count, dropped) + check.report_issue( + id=issue_id, + issue_name=ISSUE_NAME, + issue_type=ISSUE_TYPE, + title=title, + description=description, + category='integration', + severity=_severity(check, ratio), + extra={ + 'check_name': check.name, + 'endpoints': list(endpoints), + 'effective_limit': limit, + 'observed_contexts': observed_count, + 'dropped_contexts': dropped, + 'dropped_ratio': round(ratio, 4), + 'limit_is_default': limit == check.DEFAULT_METRIC_LIMIT, + }, + remediation=_remediation(filter_option_text=self.filter_option_text), + tags=[f'integration:{check.name}', 'openmetrics', 'metric-limit'], + ) + return + + check.resolve_issue(issue_id) + + +def _normalize_endpoints(endpoints: Iterable[str | None]) -> tuple[str, ...]: + return tuple(sorted({endpoint for endpoint in endpoints if endpoint})) + + +def _issue_id(hostname: str, check_name: str, endpoints: tuple[str, ...], namespace: object) -> str: + # Keep the original identity for one endpoint while representing multiple endpoints structurally. + endpoint_identity: str | list[str] = endpoints[0] if len(endpoints) == 1 else list(endpoints) + identity = json.dumps((hostname, check_name, endpoint_identity, str(namespace)), separators=(',', ':')) + digest = hashlib.sha256(identity.encode('utf-8')).hexdigest()[:16] + return f'openmetrics-dropped-config:{digest}' + + +def _issue_text( + check_name: str, endpoints: tuple[str, ...], limit: int, observed_count: int, dropped: int +) -> tuple[str, str]: + endpoint_count = len(endpoints) + endpoint_noun = 'endpoint' if endpoint_count == 1 else 'endpoints' + title = f'Dropping {dropped} of {observed_count} OpenMetrics metrics' + description = ( + f'The {check_name} check collected {observed_count} metric contexts from {endpoint_count} configured ' + f'{endpoint_noun}: {", ".join(endpoints)}. These totals cover the complete check run. The check is configured ' + f'to submit at most {limit} metric contexts per run, so the Agent submitted {limit} and discarded the ' + f'remaining {dropped}.' + ) + return title, description + + +def _severity(check: AgentCheck, ratio: float) -> int: + if ratio >= SEVERITY_HIGH_RATIO: + return check.IssueSeverity['HIGH'] + if ratio >= SEVERITY_MEDIUM_RATIO: + return check.IssueSeverity['MEDIUM'] + return check.IssueSeverity['LOW'] + + +def _remediation(*, filter_option_text: str) -> dict[str, str | list[dict[str, int | str]]]: + return { + 'summary': ( + 'Reduce the metrics submitted by this check instance, or raise its metric limit after checking the cost.' + ), + 'steps': [ + { + 'order': 1, + 'text': ( + f'Decide what you actually need. Use {filter_option_text} on this instance to stop collecting ' + 'series you do not query, alert on, or keep.' + ), + }, + { + 'order': 2, + 'text': 'Only then raise max_returned_metrics on this instance to a value above the observed count.', + }, + { + 'order': 3, + 'text': ( + 'Verify: on the instance, set metric_contexts to true under the debug_metrics section. ' + 'This publishes datadog.agent.metrics.contexts.total and ' + 'datadog.agent.metrics.contexts.limit; confirm the total stays below the limit at peak. ' + 'Consider a monitor at 80% of the limit.' + ), + }, + { + 'order': 4, + 'text': ( + 'Check the cost before you leave it: additional contexts are billable custom metrics and increase ' + 'Agent memory.' + ), + }, + ], + } diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py index 2fb5062e146bf..845ee553770bf 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py @@ -12,6 +12,7 @@ from requests.exceptions import RequestException from datadog_checks.base.checks import AgentCheck +from datadog_checks.base.checks.openmetrics.metric_limit_issue import MetricLimitIssueReporter from datadog_checks.base.errors import ConfigurationError from datadog_checks.base.utils.tracing import traced_class @@ -63,6 +64,9 @@ def __init__(self, name, init_config, instances): When overriding, make sure to call this (the parent's) __init__ first! """ super(OpenMetricsBaseCheckV2, self).__init__(name, init_config, instances) + self.metric_limit_issue_reporter: MetricLimitIssueReporter = MetricLimitIssueReporter( + filter_option_text='metrics / exclude_metrics' + ) # All desired scraper configurations, which subclasses can override as needed self.scraper_configs = [self.instance] @@ -96,6 +100,18 @@ def check(self, _): self.log.error("There was an error scraping endpoint %s: %s", endpoint, str(e)) raise type(e)("There was an error scraping endpoint {}: {}".format(endpoint, e)) from None + def _on_metric_limit_state(self, reached_limit: bool, observed_count: int, limit: int) -> None: + # Use the actual configured scraper endpoint keys rather than the raw instance field, so + # integrations that synthesize ``scraper_configs`` from options such as ``agent_endpoint`` + # (e.g. Cilium) still report drops against the endpoint that was scraped. + self.metric_limit_issue_reporter.handle( + self, + self.scrapers.keys(), + reached_limit, + observed_count, + limit, + ) + def configure_scrapers(self): """ Creates a scraper configuration for each instance. diff --git a/datadog_checks_base/tests/base/checks/openmetrics/test_metric_limit_issue.py b/datadog_checks_base/tests/base/checks/openmetrics/test_metric_limit_issue.py new file mode 100644 index 0000000000000..b1bef3162d1f5 --- /dev/null +++ b/datadog_checks_base/tests/base/checks/openmetrics/test_metric_limit_issue.py @@ -0,0 +1,296 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Any +from unittest import mock + +import pytest + +from datadog_checks.base import AgentCheck, OpenMetricsBaseCheckV2 +from datadog_checks.base.checks.openmetrics.metric_limit_issue import ISSUE_NAME, _issue_id + +ENDPOINT = 'http://example.test/metrics' +ISSUE_ID = 'openmetrics-dropped-config:5505571e531f7cf6' + + +class GenericLimitedCheck(AgentCheck): + """A non-OpenMetrics check that exceeds its metric limit. + + This verifies that AgentCheck does not report an Agent Health issue by + default; only OpenMetrics checks do so by overriding _on_metric_limit_state. + """ + + def check(self, _: Any) -> None: + for value in range(12): + self.gauge('generic.metric', value) + + +class MetricLimitOpenMetricsCheck(OpenMetricsBaseCheckV2): + def __init__(self, name: str, init_config: dict[str, Any], instances: list[dict[str, Any]]) -> None: + super().__init__(name, init_config, instances) + self.observed = 0 + + def configure_scrapers(self) -> None: + self.scrapers = {config.get('openmetrics_endpoint', ''): None for config in self.scraper_configs} + + def check(self, _: Any) -> None: + for value in range(self.observed): + self.gauge('openmetrics.metric', value) + + +class IsolatedMetricLimitOpenMetricsCheck(MetricLimitOpenMetricsCheck): + """Submits a fixed number of contexts. + + The isolated child process reconstructs the check from serialized configuration + alone, so parent-object attributes such as ``observed`` do not reach it. + """ + + def check(self, _: Any) -> None: + for value in range(20): + self.gauge('openmetrics.metric', value) + + +def create_check(limit: int = 5, endpoint: str = ENDPOINT) -> MetricLimitOpenMetricsCheck: + instance = { + 'openmetrics_endpoint': endpoint, + 'namespace': 'demo', + 'max_returned_metrics': limit, + } + return MetricLimitOpenMetricsCheck('openmetrics_test', {}, [instance]) + + +def reported_issues(datadog_agent: Any) -> list[dict[str, Any]]: + return datadog_agent._sent_reported_issues['openmetrics_test'] + + +def test_generic_agent_check_metric_limiter_does_not_report(datadog_agent: Any) -> None: + check = GenericLimitedCheck('generic', {}, [{'max_returned_metrics': 2}]) + + assert check.run() == '' + + assert not datadog_agent._sent_reported_issues + + +def test_over_limit_run_reports_expected_issue(datadog_agent: Any) -> None: + check = create_check() + check.observed = 20 + + assert check.run() == '' + + [issue] = reported_issues(datadog_agent) + assert issue['id'] == ISSUE_ID + assert issue['issue_name'] == ISSUE_NAME + assert issue['issue_type'] == 'openmetrics_metrics_dropped_by_configured_limit' + assert issue['category'] == 'integration' + assert issue['severity'] == check.IssueSeverity['HIGH'] + assert issue['extra'] == { + 'check_name': 'openmetrics_test', + 'endpoints': [ENDPOINT], + 'effective_limit': 5, + 'observed_contexts': 20, + 'dropped_contexts': 15, + 'dropped_ratio': 0.75, + 'limit_is_default': False, + } + assert issue['tags'] == ['integration:openmetrics_test', 'openmetrics', 'metric-limit'] + assert len(issue['remediation']['steps']) == 4 + + verify_step = issue['remediation']['steps'][2]['text'] + # The Fleet UI renders remediation text as plain text, so the config key must + # be described as nested (not as a dotted single-line key, which the check does + # not parse) and both emitted metric names must be spelled out in full. + assert 'debug_metrics.metric_contexts: true' not in verify_step + assert 'metric_contexts to true under the debug_metrics section' in verify_step + assert 'datadog.agent.metrics.contexts.total' in verify_step + assert 'datadog.agent.metrics.contexts.limit' in verify_step + + +def test_repeated_over_limit_runs_report_same_id(datadog_agent: Any) -> None: + check = create_check() + check.observed = 20 + + check.run() + check.run() + + assert [issue['id'] for issue in reported_issues(datadog_agent)] == [ISSUE_ID, ISSUE_ID] + + +def test_one_dropped_metric_reports_low_severity(datadog_agent: Any) -> None: + check = create_check(limit=100) + check.observed = 101 + + check.run() + + [issue] = reported_issues(datadog_agent) + assert issue['severity'] == check.IssueSeverity['LOW'] + assert issue['extra']['dropped_contexts'] == 1 + + +@pytest.mark.parametrize( + ('observed', 'limit', 'severity'), + [ + pytest.param(1000, 990, 'LOW', id='low'), + pytest.param(200, 190, 'MEDIUM', id='medium'), + pytest.param(40, 30, 'HIGH', id='high'), + ], +) +def test_severity_thresholds(datadog_agent: Any, observed: int, limit: int, severity: str) -> None: + check = create_check(limit=limit) + check.observed = observed + + check.run() + + [issue] = reported_issues(datadog_agent) + assert issue['severity'] == check.IssueSeverity[severity] + + +def test_resolves_on_first_clean_run(datadog_agent: Any) -> None: + check = create_check() + check.observed = 20 + check.run() + + check.observed = 5 + check.run() + + assert datadog_agent._sent_resolved_issues == [ISSUE_ID] + + +def test_clean_runs_resolve_idempotently(datadog_agent: Any) -> None: + check = create_check() + check.observed = 5 + + check.run() + check.run() + + assert datadog_agent._sent_resolved_issues == [ISSUE_ID, ISSUE_ID] + + +def test_recurrence_after_resolution_reports_same_id(datadog_agent: Any) -> None: + check = create_check() + check.observed = 20 + check.run() + + check.observed = 5 + check.run() + + check.observed = 20 + check.run() + + assert [issue['id'] for issue in reported_issues(datadog_agent)] == [ISSUE_ID, ISSUE_ID] + assert datadog_agent._sent_resolved_issues == [ISSUE_ID] + + +def test_failed_run_does_not_resolve_active_issue(datadog_agent: Any) -> None: + check = create_check() + check.observed = 20 + check.run() + + check.check = mock.Mock(side_effect=RuntimeError('scrape failure')) + error_report = check.run() + + assert 'scrape failure' in error_report + assert datadog_agent._sent_resolved_issues == [] + + +@pytest.mark.parametrize( + ('hostname', 'check_name', 'endpoint', 'namespace'), + [ + pytest.param('other.hostname', 'openmetrics_test', ENDPOINT, 'demo', id='hostname'), + pytest.param('stubbed.hostname', 'other_openmetrics_test', ENDPOINT, 'demo', id='check-name'), + pytest.param( + 'stubbed.hostname', 'openmetrics_test', 'http://other.example.test/metrics', 'demo', id='endpoint' + ), + pytest.param('stubbed.hostname', 'openmetrics_test', ENDPOINT, 'other', id='namespace'), + ], +) +def test_issue_id_changes_with_identity_component( + hostname: str, check_name: str, endpoint: str, namespace: str +) -> None: + base_id = _issue_id('stubbed.hostname', 'openmetrics_test', (ENDPOINT,), 'demo') + + assert _issue_id(hostname, check_name, (endpoint,), namespace) != base_id + + +def test_issue_id_does_not_include_metric_limit(datadog_agent: Any) -> None: + first_check = create_check(limit=5) + first_check.observed = 20 + first_check.run() + first_id = reported_issues(datadog_agent)[0]['id'] + + second_check = create_check(limit=10) + second_check.observed = 25 + second_check.run() + second_id = reported_issues(datadog_agent)[1]['id'] + + assert first_check.metric_limiter.limit != second_check.metric_limiter.limit + assert first_id == second_id == ISSUE_ID + + +def test_report_issue_failure_does_not_break_limiter_reset_or_next_run(aggregator: Any) -> None: + check = create_check(limit=2) + check.observed = 12 + check.report_issue = mock.Mock(side_effect=RuntimeError('bridge failure')) + + assert check.run() == '' + assert check.metric_limiter.reached_limit is False + assert len(aggregator.metrics('openmetrics.metric')) == 2 + + check.observed = 2 + assert check.run() == '' + assert check.metric_limiter.reached_limit is False + assert len(aggregator.metrics('openmetrics.metric')) == 4 + + +def test_missing_endpoint_does_nothing(datadog_agent: Any, caplog: Any) -> None: + check = create_check(endpoint='') + check.observed = 20 + + with caplog.at_level('DEBUG'): + check.run() + + assert not datadog_agent._sent_reported_issues + assert datadog_agent._sent_resolved_issues == [] + assert 'without an endpoint' in caplog.text + + +def test_isolated_parent_does_not_invoke_metric_limit_state(datadog_agent: Any) -> None: + check = create_check() + check.instance['process_isolation'] = True + check._on_metric_limit_state = mock.Mock() + + with mock.patch('datadog_checks.base.utils.replay.execute.run_with_isolation'): + assert check.run() == '' + + check._on_metric_limit_state.assert_not_called() + + +def test_isolated_check_reports_metric_limit_issue(datadog_agent: Any) -> None: + """The over-limit condition is observed by the isolated child process, whose + report_issue call is replayed back to this process through the Agent stub.""" + check = IsolatedMetricLimitOpenMetricsCheck( + 'openmetrics_test', + {}, + [ + { + 'openmetrics_endpoint': ENDPOINT, + 'namespace': 'demo', + 'max_returned_metrics': 5, + 'process_isolation': True, + } + ], + ) + check.check_id = 'test:123' + + assert check.run() == '' + + # The parent never submitted metrics, so its limiter stayed untouched; the + # issue below could only have been reported by the isolated child process. + assert check.metric_limiter.count == 0 + + [issue] = reported_issues(datadog_agent) + assert issue['id'] == ISSUE_ID + assert issue['issue_name'] == ISSUE_NAME + assert issue['issue_type'] == 'openmetrics_metrics_dropped_by_configured_limit' + assert issue['extra']['observed_contexts'] == 20 + assert issue['extra']['dropped_contexts'] == 15 + assert datadog_agent._sent_resolved_issues == [] diff --git a/datadog_checks_base/tests/base/checks/test_agent_check.py b/datadog_checks_base/tests/base/checks/test_agent_check.py index 8cb0e6a8c1c0b..ecd34537013b5 100644 --- a/datadog_checks_base/tests/base/checks/test_agent_check.py +++ b/datadog_checks_base/tests/base/checks/test_agent_check.py @@ -1639,7 +1639,7 @@ def issue_check(): def test_report_issue_minimal(datadog_agent, issue_check): - issue_check.report_issue(id="issue-1", issue_name="connection_failed") + issue_check.report_issue(id="issue-1", issue_name="connection_failed", issue_type="connection_failed") datadog_agent.assert_reported_issue( "test_check", @@ -1647,6 +1647,7 @@ def test_report_issue_minimal(datadog_agent, issue_check): { 'id': 'issue-1', 'issue_name': 'connection_failed', + 'issue_type': 'connection_failed', 'title': None, 'description': None, 'category': None, @@ -1663,7 +1664,8 @@ def test_report_issue_minimal(datadog_agent, issue_check): def test_report_issue_full(datadog_agent, issue_check): issue_check.report_issue( id="issue-2", - issue_name="permission_denied", + issue_name="Permission Denied", + issue_type="permission_denied", title="Permission denied", description="The check lacks required permissions.", category="permissions", @@ -1681,7 +1683,8 @@ def test_report_issue_full(datadog_agent, issue_check): "issue-2", { 'id': 'issue-2', - 'issue_name': 'permission_denied', + 'issue_name': 'Permission Denied', + 'issue_type': 'permission_denied', 'title': 'Permission denied', 'description': 'The check lacks required permissions.', 'category': 'permissions', @@ -1699,15 +1702,16 @@ def test_report_issue_full(datadog_agent, issue_check): @pytest.mark.parametrize( - 'issue_id, issue_name, expected_message', + 'issue_id, issue_name, issue_type, expected_message', [ - pytest.param('', 'connection_failed', 'Issue ID is required', id='missing id'), - pytest.param('issue-1', '', 'Issue Name is required', id='missing issue name'), + pytest.param('', 'connection_failed', 'connection_failed', 'Issue ID is required', id='missing id'), + pytest.param('issue-1', '', 'connection_failed', 'Issue Name is required', id='missing issue name'), + pytest.param('issue-1', 'connection_failed', '', 'Issue Type is required', id='missing issue type'), ], ) -def test_report_issue_requires_id_and_name(issue_check, issue_id, issue_name, expected_message): +def test_report_issue_requires_required_fields(issue_check, issue_id, issue_name, issue_type, expected_message): with pytest.raises(ValueError, match=expected_message): - issue_check.report_issue(id=issue_id, issue_name=issue_name) + issue_check.report_issue(id=issue_id, issue_name=issue_name, issue_type=issue_type) def test_resolve_issue(datadog_agent, issue_check): diff --git a/openmetrics/tests/test_metric_limit_issue.py b/openmetrics/tests/test_metric_limit_issue.py new file mode 100644 index 0000000000000..2a5307c4bbff5 --- /dev/null +++ b/openmetrics/tests/test_metric_limit_issue.py @@ -0,0 +1,59 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Any + +import pytest + +from datadog_checks.base.checks.openmetrics.metric_limit_issue import ISSUE_NAME +from datadog_checks.openmetrics import OpenMetricsCheck + +ISSUE_ID = 'openmetrics-dropped-config:40c8930ce3bf6455' +ENDPOINT = 'http://localhost:10249/metrics' + + +@pytest.mark.parametrize( + ('instance', 'filter_option'), + [ + pytest.param( + { + 'openmetrics_endpoint': ENDPOINT, + 'namespace': 'openmetrics', + 'metrics': ['.*'], + 'max_returned_metrics': 5, + }, + 'metrics / exclude_metrics', + id='v2', + ), + pytest.param( + { + 'prometheus_url': ENDPOINT, + 'namespace': 'openmetrics', + 'metrics': ['*'], + 'max_returned_metrics': 5, + }, + 'metrics / ignore_metrics', + id='v1', + ), + ], +) +def test_openmetrics_base_classes_report_metric_limit_issue( + datadog_agent: Any, instance: dict[str, Any], filter_option: str +) -> None: + check = OpenMetricsCheck('openmetrics', {}, [instance]) + # Calling the callback directly skips V2 scraper setup. Populate the endpoint + # state that configure_scrapers() normally creates. V1 gets its endpoint + # directly from prometheus_url, so it needs no equivalent setup. + if 'openmetrics_endpoint' in instance: + check.scrapers = {instance['openmetrics_endpoint']: None} + + check._on_metric_limit_state(True, 20, 5) + + [issue] = datadog_agent._sent_reported_issues['openmetrics'] + assert issue['id'] == ISSUE_ID + assert issue['issue_name'] == ISSUE_NAME + assert issue['issue_type'] == 'openmetrics_metrics_dropped_by_configured_limit' + assert issue['extra']['endpoints'] == [ENDPOINT] + assert issue['extra']['observed_contexts'] == 20 + assert issue['extra']['dropped_contexts'] == 15 + assert filter_option in issue['remediation']['steps'][0]['text']