feat(otel): otel sampling support in distributed tracing #10117
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| 'use strict' | ||
|
|
||
| const UINT64_MODULO = 2n ** 64n | ||
|
|
||
| // Knuth's factor for the sampling algorithm shared across Datadog tracers. | ||
| const SAMPLING_KNUTH_FACTOR = 1_111_111_111_111_111_111n | ||
|
|
||
| /** | ||
| * Hashes the lower 64 bits of a trace ID for deterministic trace sampling. | ||
| * | ||
| * @param {bigint} traceId | ||
| * @returns {bigint} | ||
| */ | ||
| function knuthHash (traceId) { | ||
| return (traceId * SAMPLING_KNUTH_FACTOR) % UINT64_MODULO | ||
| } | ||
|
|
||
| module.exports = knuthHash |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -204,8 +204,8 @@ class DatadogSpan { | |
| setTag (key, value) { | ||
| this._spanContext.setTag(key, value) | ||
|
|
||
| if (isSamplingPriorityTag(key)) { | ||
| this._prioritySampler.setPriorityFromTag(this, key, value) | ||
|
Comment on lines
+216
to
+217
There was a problem hiding this comment.
After sampling has already been materialized—for example by injection or reading OTel AGENTS.md reference: AGENTS.md:L109-L109 Useful? React with 👍 / 👎. |
||
| } | ||
|
|
||
| if (tagsUpdateCh.hasSubscribers) { | ||
|
|
@@ -223,9 +223,11 @@ class DatadogSpan { | |
| // `options.tags` callers that pass `'key:val,key:val'` strings. | ||
| const tags = this._spanContext.getTags() | ||
| let mayChangeSamplingPriority | ||
| let samplingTags | ||
|
|
||
| if (keyValueMap !== null && typeof keyValueMap === 'object' && !Array.isArray(keyValueMap)) { | ||
| Object.assign(tags, keyValueMap) | ||
| samplingTags = keyValueMap | ||
| mayChangeSamplingPriority = | ||
| MANUAL_KEEP in keyValueMap || | ||
| MANUAL_DROP in keyValueMap || | ||
|
|
@@ -234,14 +236,15 @@ class DatadogSpan { | |
| /* istanbul ignore if: v5 fallback, master ships 6.0.0-pre */ | ||
| if (DD_MAJOR < 6 && (typeof keyValueMap === 'string' || Array.isArray(keyValueMap))) { | ||
| tagger.add(tags, keyValueMap) | ||
| samplingTags = tags | ||
| mayChangeSamplingPriority = true | ||
|
There was a problem hiding this comment.
With AGENTS.md reference: AGENTS.md:L119-L122 Useful? React with 👍 / 👎. |
||
| } else { | ||
| return this | ||
| } | ||
| } | ||
|
|
||
| if (mayChangeSamplingPriority) { | ||
| this._prioritySampler.setPriorityFromTags(this, samplingTags) | ||
| } | ||
|
|
||
| if (tagsUpdateCh.hasSubscribers) { | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,198 @@ | ||
| 'use strict' | ||
|
|
||
| const { AUTO_KEEP } = require('../../../ext/priority') | ||
| const knuthHash = require('./knuth_hash') | ||
| const { SAMPLING_AGENT_DECISION, SAMPLING_RULE_DECISION } = require('./constants') | ||
|
|
||
| const MAX_OTEL_VALUE_BYTES = 256 | ||
| const MAX_THRESHOLD = 2n ** 56n | ||
| const MAX_ENCODABLE_THRESHOLD = MAX_THRESHOLD - 1n | ||
| const UINT64_MASK = 2n ** 64n - 1n | ||
| const validRandomValue = /^[0-9a-f]{14}$/ | ||
| const validThreshold = /^[0-9a-f]{1,14}$/ | ||
|
|
||
| /** | ||
| * Derives the OTel 56-bit random value from Datadog's sampling hash. | ||
| * | ||
| * @param {bigint} traceId | ||
| * @returns {bigint} | ||
| */ | ||
| function randomValueFor (traceId) { | ||
| return ((~knuthHash(traceId)) & UINT64_MASK) >> 8n | ||
| } | ||
|
|
||
| /** | ||
| * Converts a sample rate to an OTel 56-bit rejection threshold. | ||
| * | ||
| * @param {number} sampleRate | ||
| * @returns {bigint} | ||
| */ | ||
| function thresholdFor (sampleRate) { | ||
| if (sampleRate === 1) return 0n | ||
| if (sampleRate === 0) return MAX_ENCODABLE_THRESHOLD | ||
|
|
||
| const threshold = BigInt(Math.round((1 - sampleRate) * Number(MAX_THRESHOLD))) | ||
| if (threshold < 0n) return 0n | ||
| if (threshold > MAX_ENCODABLE_THRESHOLD) return MAX_ENCODABLE_THRESHOLD | ||
| return threshold | ||
| } | ||
|
|
||
| /** | ||
| * Formats an OTel threshold with trailing zero nibbles removed. | ||
| * | ||
| * @param {bigint} threshold | ||
| * @returns {string} | ||
| */ | ||
| function formatThreshold (threshold) { | ||
| return threshold.toString(16).padStart(14, '0').replace(/0+$/, '') || '0' | ||
| } | ||
|
|
||
| /** | ||
| * Generates OTel sampling fields for a local probability decision. | ||
| * | ||
| * @param {import('./opentracing/span_context')} context | ||
| * @returns {{ randomValue: string, threshold: string } | undefined} | ||
| */ | ||
| function generateFields (context) { | ||
| const { priority } = context._sampling | ||
| const probabilityRate = getProbabilityRate(context) | ||
| if (priority === undefined || probabilityRate === undefined) return | ||
|
|
||
| const thresholdValue = thresholdFor(probabilityRate) | ||
| let randomValue = randomValueFor(context._traceId.toBigInt()) | ||
| const kept = priority >= AUTO_KEEP | ||
|
|
||
| if (kept && randomValue < thresholdValue) { | ||
| randomValue = thresholdValue | ||
| } else if (!kept && randomValue >= thresholdValue) { | ||
| randomValue = thresholdValue > 0n ? thresholdValue - 1n : 0n | ||
| } | ||
|
|
||
| return { | ||
| randomValue: randomValue.toString(16).padStart(14, '0'), | ||
| threshold: formatThreshold(thresholdValue), | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Returns the probability rate already recorded by the regular Datadog sampling path. | ||
| * | ||
| * @param {import('./opentracing/span_context')} context | ||
| * @returns {number | undefined} | ||
| */ | ||
| function getProbabilityRate (context) { | ||
| if (context._sampling.isProbabilityDecision === false) return | ||
| if (context._sampling.probabilityRate !== undefined) return context._sampling.probabilityRate | ||
| return context._trace[SAMPLING_RULE_DECISION] ?? context._trace[SAMPLING_AGENT_DECISION] | ||
| } | ||
|
|
||
| /** | ||
| * Adds a complete sub-field while the OTel member remains within its byte cap. | ||
| * | ||
| * @param {string[]} fields | ||
| * @param {string} field | ||
| * @param {number} byteLength | ||
| * @returns {number} | ||
| */ | ||
| function addField (fields, field, byteLength) { | ||
| const fieldLength = Buffer.byteLength(field) + (fields.length === 0 ? 0 : 1) | ||
| if (byteLength + fieldLength <= MAX_OTEL_VALUE_BYTES) { | ||
| fields.push(field) | ||
| return byteLength + fieldLength | ||
| } | ||
| return byteLength | ||
| } | ||
|
|
||
| /** | ||
| * Parses and rebuilds the OTel tracestate member, preserving unknown sub-fields. | ||
| * | ||
| * @param {import('./opentracing/span_context')} context | ||
| * @param {string | undefined} member | ||
| * @returns {string | undefined} | ||
| */ | ||
| function buildOtelMember (context, member) { | ||
| if (member === undefined) { | ||
| if (context._sampling.isProbabilityDecision === false) return | ||
| const generated = generateFields(context) | ||
| if (!generated) return | ||
| return `rv:${generated.randomValue};th:${generated.threshold}` | ||
| } | ||
|
|
||
| let randomValue | ||
| let threshold | ||
| const unknownFields = [] | ||
| let start = 0 | ||
|
|
||
| while (start <= member.length) { | ||
| let end = member.indexOf(';', start) | ||
| if (end === -1) end = member.length | ||
| const field = member.slice(start, end) | ||
| if (field) { | ||
| const separator = field.indexOf(':') | ||
| const key = separator === -1 ? field : field.slice(0, separator) | ||
| const value = separator === -1 ? undefined : field.slice(separator + 1) | ||
| if (key === 'rv') { | ||
| randomValue = value | ||
| } else if (key === 'th') { | ||
| threshold = value | ||
| } else { | ||
| unknownFields.push(field) | ||
| } | ||
| } | ||
| if (end === member.length) break | ||
| start = end + 1 | ||
| } | ||
|
|
||
| if (!validRandomValue.test(randomValue)) randomValue = undefined | ||
| if (!validThreshold.test(threshold)) threshold = undefined | ||
|
|
||
| if (context._sampling.isProbabilityDecision === false) { | ||
| threshold = undefined | ||
| } else if (randomValue === undefined && threshold === undefined) { | ||
| const generated = generateFields(context) | ||
| if (generated) { | ||
| randomValue = generated.randomValue | ||
| threshold = generated.threshold | ||
| } | ||
| } | ||
|
|
||
| const fields = [] | ||
| let byteLength = 0 | ||
| if (randomValue !== undefined) byteLength = addField(fields, `rv:${randomValue}`, byteLength) | ||
| if (threshold !== undefined) byteLength = addField(fields, `th:${threshold}`, byteLength) | ||
| for (const field of unknownFields) { | ||
| byteLength = addField(fields, field, byteLength) | ||
| } | ||
|
|
||
| return fields.length === 0 ? undefined : fields.join(';') | ||
| } | ||
|
|
||
| /** | ||
| * Updates the OTel tracestate member to represent the context's sampling decision. | ||
| * | ||
| * @param {import('./opentracing/span_context')} context | ||
| * @param {import('./opentracing/propagation/tracestate')} traceState | ||
| * @returns {void} | ||
| */ | ||
| function updateOtelTraceState (context, traceState) { | ||
| const otelMember = traceState.get('ot') | ||
| if (context._sampling.isProbabilityDecision === false) { | ||
| if (otelMember === undefined) return | ||
| } else if (getProbabilityRate(context) === undefined) { | ||
| // Reinsert the inherited member to keep it leftmost without rebuilding its fields. | ||
| if (otelMember !== undefined) traceState.set('ot', otelMember) | ||
| return | ||
| } | ||
|
|
||
| const rebuiltOtelMember = buildOtelMember(context, otelMember) | ||
| if (rebuiltOtelMember === undefined) { | ||
| traceState.delete('ot') | ||
| } else { | ||
| traceState.set('ot', rebuiltOtelMember) | ||
| } | ||
| } | ||
|
|
||
| module.exports = { | ||
| buildOtelMember, | ||
| updateOtelTraceState, | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.