diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4c1d2240fd8..a11d4486545 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -328,6 +328,9 @@ /packages/dd-trace/test/baggage.spec.js @DataDog/dd-trace-js @DataDog/apm-sdk-capabilities-js /packages/dd-trace/src/carrier.js @DataDog/dd-trace-js @DataDog/apm-sdk-capabilities-js /packages/dd-trace/test/carrier.spec.js @DataDog/dd-trace-js @DataDog/apm-sdk-capabilities-js +/packages/dd-trace/src/knuth-hash.js @DataDog/dd-trace-js @DataDog/apm-sdk-capabilities-js +/packages/dd-trace/src/otel-sampling.js @DataDog/dd-trace-js @DataDog/apm-sdk-capabilities-js +/packages/dd-trace/test/opentracing/propagation/otel-sampling.spec.js @DataDog/dd-trace-js @DataDog/apm-sdk-capabilities-js /packages/dd-trace/src/*sampler.js @DataDog/dd-trace-js @DataDog/apm-sdk-capabilities-js /packages/dd-trace/test/*sampler.spec.js @DataDog/dd-trace-js @DataDog/apm-sdk-capabilities-js /packages/dd-trace/src/sampling_rule.js @DataDog/dd-trace-js @DataDog/apm-sdk-capabilities-js diff --git a/packages/dd-trace/src/knuth-hash.js b/packages/dd-trace/src/knuth-hash.js new file mode 100644 index 00000000000..26e111c27d1 --- /dev/null +++ b/packages/dd-trace/src/knuth-hash.js @@ -0,0 +1,18 @@ +'use strict' + +const UINT64_MODULO = 2n ** 64n + +// Knuth's factor for the sampling algorithm shared across Datadog tracers. +const SAMPLING_KNUTH_FACTOR = 1_111_111_111_111_111_111n + +/** + * Hashes the lower 64 bits of a trace ID for deterministic trace sampling. + * + * @param {bigint} traceId + * @returns {bigint} + */ +function knuthHash (traceId) { + return (traceId * SAMPLING_KNUTH_FACTOR) % UINT64_MODULO +} + +module.exports = knuthHash diff --git a/packages/dd-trace/src/opentelemetry/span_context.js b/packages/dd-trace/src/opentelemetry/span_context.js index d2ea50fcb9a..68dce79c42e 100644 --- a/packages/dd-trace/src/opentelemetry/span_context.js +++ b/packages/dd-trace/src/opentelemetry/span_context.js @@ -2,7 +2,9 @@ const api = require('@opentelemetry/api') const { AUTO_KEEP } = require('../../../../ext/priority') +const { updateOtelTraceState } = require('../otel-sampling') const DatadogSpanContext = require('../opentracing/span_context') +const TraceState = require('../opentracing/propagation/tracestate') const id = require('../id') function newContext () { @@ -37,8 +39,10 @@ class SpanContext { } get traceState () { - const ts = this._ddContext._tracestate - return api.createTraceState(ts ? ts.toString() : '') + this._ddContext._ensureSamplingPriority() + const traceState = TraceState.fromString(this._ddContext._tracestate?.toString()) + updateOtelTraceState(this._ddContext, traceState) + return api.createTraceState(traceState.toString()) } } diff --git a/packages/dd-trace/src/opentelemetry/tracer.js b/packages/dd-trace/src/opentelemetry/tracer.js index 3f15c315fbd..e790e8b8917 100644 --- a/packages/dd-trace/src/opentelemetry/tracer.js +++ b/packages/dd-trace/src/opentelemetry/tracer.js @@ -74,12 +74,10 @@ class Tracer { _convertOtelContextToDatadog (traceId, spanId, traceFlag, ts, meta = {}) { let origin = null let samplingPriority = traceFlag + const traceStateValue = typeof ts?.serialize === 'function' ? ts.serialize() : ts?.traceparent + const traceState = TraceState.fromString(traceStateValue) - ts = ts?.traceparent - - if (ts) { - // Use TraceState.fromString to parse the tracestate header - const traceState = TraceState.fromString(ts) + if (traceStateValue) { let ddTraceStateData = null // Extract Datadog specific trace state data @@ -102,12 +100,12 @@ class Tracer { const tracestateSamplingPriority = samplingPriorityTs ? Math.trunc(samplingPriorityTs) : undefined samplingPriority = getSamplingPriority(traceFlag, tracestateSamplingPriority, origin) } else { - log.debug('No dd list member in tracestate from incoming request:', ts) + log.debug('No dd list member in tracestate from incoming request:', traceStateValue) } } const spanContext = new SpanContext({ - traceId: id(traceId, 16), spanId: id(), tags: meta, parentId: id(spanId, 16), + traceId: id(traceId, 16), spanId: id(), tags: meta, parentId: id(spanId, 16), tracestate: traceState, }) spanContext._ddContext._sampling = { priority: samplingPriority } diff --git a/packages/dd-trace/src/opentracing/propagation/text_map.js b/packages/dd-trace/src/opentracing/propagation/text_map.js index 3442d4825dd..d7eeac82660 100644 --- a/packages/dd-trace/src/opentracing/propagation/text_map.js +++ b/packages/dd-trace/src/opentracing/propagation/text_map.js @@ -77,6 +77,8 @@ const zeroTraceId = '0000000000000000' const hex16 = /^[0-9A-Fa-f]{16}$/ const percentByte = /%([0-9A-Fa-f]{2})/g +let updateOtelTraceState + /** * @typedef {object} B3Context * @property {string} [flags] @@ -151,12 +153,15 @@ function getB3Priority (sampled, debug) { * @returns {DatadogSpanContext | undefined} */ function extractB3Context (b3) { - const priority = getB3Priority(b3.sampled, b3.flags === '1') - const spanContext = extractGenericContext(b3.traceId, b3.spanId, 16) + const debug = b3.flags === '1' + const priority = getB3Priority(b3.sampled, debug) + let spanContext = extractGenericContext(b3.traceId, b3.spanId, 16) if (priority !== undefined) { - if (!spanContext) { - return new DatadogSpanContext({ + if (spanContext) { + spanContext._sampling.priority = priority + } else { + spanContext = new DatadogSpanContext({ traceId: id(), spanId: null, sampling: { priority }, @@ -164,7 +169,7 @@ function extractB3Context (b3) { }) } - spanContext._sampling.priority = priority + if (debug) spanContext._sampling.isProbabilityDecision = false } if (spanContext && b3.traceId) extract128BitTraceId(b3.traceId, spanContext) @@ -619,6 +624,9 @@ class TextMapPropagator { writeTraceparent(carrier, spanContext.toTraceparent()) + updateOtelTraceState ??= require('../../otel-sampling').updateOtelTraceState + updateOtelTraceState(spanContext, ts) + ts.forVendor('dd', state => { if (!spanContext._isRemote) { // SpanContext was created by a ddtrace span. @@ -703,10 +711,25 @@ class TextMapPropagator { */ #resolveTraceContextConflicts (w3cSpanContext, firstSpanContext, carrier, datadogContext) { if (w3cSpanContext === undefined || - firstSpanContext.toTraceId(true) !== w3cSpanContext.toTraceId(true) || - firstSpanContext.toSpanId() === w3cSpanContext.toSpanId()) { + firstSpanContext.toTraceId(true) !== w3cSpanContext.toTraceId(true)) { return firstSpanContext } + + const selectedPriority = firstSpanContext._sampling.priority + if (selectedPriority === undefined) { + firstSpanContext._sampling.priority = w3cSpanContext._sampling.priority + } else if ((selectedPriority >= AUTO_KEEP) !== (w3cSpanContext._sampling.priority >= AUTO_KEEP)) { + // The W3C threshold describes its sampled bit, not the conflicting decision selected from another style. + firstSpanContext._sampling.isProbabilityDecision = false + // The copied decision maker likewise belongs to the conflicting W3C decision. + w3cSpanContext._tracestate.forVendor('dd', state => { + if (state.get('t.dm') !== undefined) state.delete('t.dm') + }) + } + + firstSpanContext._tracestate = w3cSpanContext._tracestate + if (firstSpanContext.toSpanId() === w3cSpanContext.toSpanId()) return firstSpanContext + if (tags.DD_PARENT_ID in w3cSpanContext._trace.tags) { // tracecontext headers contain a p value, ensure this value is sent to backend firstSpanContext._trace.tags[tags.DD_PARENT_ID] = w3cSpanContext._trace.tags[tags.DD_PARENT_ID] diff --git a/packages/dd-trace/src/opentracing/propagation/tracestate.js b/packages/dd-trace/src/opentracing/propagation/tracestate.js index 39612d3a9b0..8261bba8a90 100644 --- a/packages/dd-trace/src/opentracing/propagation/tracestate.js +++ b/packages/dd-trace/src/opentracing/propagation/tracestate.js @@ -3,6 +3,7 @@ // W3C Trace Context §3.3.1.2: max 32 list-members. // https://www.w3.org/TR/trace-context/#tracestate-header-field-values const MAX_LIST_MEMBERS = 32 +const MAX_TRACESTATE_BYTES = 512 const WHITESPACE = /[ \t]/ /** @@ -56,6 +57,32 @@ function toString (map, pairSeparator, fieldSeparator) { return result } +/** + * Keeps complete leftmost members within the W3C count and byte limits. + * + * @param {string} value + * @returns {string} + */ +function limitTraceState (value) { + let byteLength = 0 + let end = 0 + let members = 0 + let start = 0 + + while (start < value.length && members < MAX_LIST_MEMBERS) { + let next = value.indexOf(',', start) + if (next === -1) next = value.length + const memberLength = Buffer.byteLength(value.slice(start, next)) + (members === 0 ? 0 : 1) + if (byteLength + memberLength > MAX_TRACESTATE_BYTES) break + byteLength += memberLength + end = next + members++ + start = next + 1 + } + + return value.slice(0, end) +} + class TraceStateData { #map changed = false @@ -162,7 +189,12 @@ class TraceState { } toString () { - return toString(this, '=', ',') + const value = toString(this, '=', ',') + if (this.size <= MAX_LIST_MEMBERS && + (value.length <= MAX_TRACESTATE_BYTES / 4 || Buffer.byteLength(value) <= MAX_TRACESTATE_BYTES)) { + return value + } + return limitTraceState(value) } } diff --git a/packages/dd-trace/src/opentracing/span.js b/packages/dd-trace/src/opentracing/span.js index 202650c607d..8fff4a3b79e 100644 --- a/packages/dd-trace/src/opentracing/span.js +++ b/packages/dd-trace/src/opentracing/span.js @@ -213,8 +213,8 @@ class DatadogSpan { setTag (key, value) { this._spanContext.setTag(key, value) - if (isSamplingPriorityTag(key) && this._spanContext._sampling.priority === undefined) { - this._prioritySampler.sample(this, false) + if (isSamplingPriorityTag(key)) { + this._prioritySampler.setPriorityFromTag(this, key, value) } if (tagsUpdateCh.hasSubscribers) { @@ -232,9 +232,11 @@ class DatadogSpan { // `options.tags` callers that pass `'key:val,key:val'` strings. const tags = this._spanContext.getTags() let mayChangeSamplingPriority + let samplingTags if (keyValueMap !== null && typeof keyValueMap === 'object' && !Array.isArray(keyValueMap)) { Object.assign(tags, keyValueMap) + samplingTags = keyValueMap mayChangeSamplingPriority = MANUAL_KEEP in keyValueMap || MANUAL_DROP in keyValueMap || @@ -242,15 +244,20 @@ class DatadogSpan { } else { /* istanbul ignore if: v5 fallback, master ships 6.0.0-pre */ if (DD_MAJOR < 6 && (typeof keyValueMap === 'string' || Array.isArray(keyValueMap))) { - tagger.add(tags, keyValueMap) - mayChangeSamplingPriority = true + samplingTags = {} + tagger.add(samplingTags, keyValueMap) + Object.assign(tags, samplingTags) + mayChangeSamplingPriority = + MANUAL_KEEP in samplingTags || + MANUAL_DROP in samplingTags || + SAMPLING_PRIORITY in samplingTags } else { return this } } - if (mayChangeSamplingPriority && this._spanContext._sampling.priority === undefined) { - this._prioritySampler.sample(this, false) + if (mayChangeSamplingPriority) { + this._prioritySampler.setPriorityFromTags(this, samplingTags) } if (tagsUpdateCh.hasSubscribers) { diff --git a/packages/dd-trace/src/otel-sampling.js b/packages/dd-trace/src/otel-sampling.js new file mode 100644 index 00000000000..0da778b4dc7 --- /dev/null +++ b/packages/dd-trace/src/otel-sampling.js @@ -0,0 +1,196 @@ +'use strict' + +const { AUTO_KEEP } = require('../../../ext/priority') +const knuthHash = require('./knuth-hash') +const { SAMPLING_AGENT_DECISION, SAMPLING_RULE_DECISION } = require('./constants') + +const MAX_OTEL_VALUE_BYTES = 256 +const MAX_THRESHOLD = 2n ** 56n +const MAX_ENCODABLE_THRESHOLD = MAX_THRESHOLD - 1n +const UINT64_MASK = 2n ** 64n - 1n +const validRandomValue = /^[0-9a-f]{14}$/ +const validThreshold = /^[0-9a-f]{1,14}$/ + +/** + * Derives the OTel 56-bit random value from Datadog's sampling hash. + * + * @param {bigint} traceId + * @returns {bigint} + */ +function randomValueFor (traceId) { + return ((~knuthHash(traceId)) & UINT64_MASK) >> 8n +} + +/** + * Converts a sample rate to an OTel 56-bit rejection threshold. + * + * @param {number} sampleRate + * @returns {bigint} + */ +function thresholdFor (sampleRate) { + if (sampleRate === 1) return 0n + if (sampleRate === 0) return MAX_ENCODABLE_THRESHOLD + + const threshold = BigInt(Math.round((1 - sampleRate) * Number(MAX_THRESHOLD))) + if (threshold < 0n) return 0n + if (threshold > MAX_ENCODABLE_THRESHOLD) return MAX_ENCODABLE_THRESHOLD + return threshold +} + +/** + * Formats an OTel threshold with trailing zero nibbles removed. + * + * @param {bigint} threshold + * @returns {string} + */ +function formatThreshold (threshold) { + return threshold.toString(16).padStart(14, '0').replace(/0+$/, '') || '0' +} + +/** + * Generates OTel sampling fields for a local probability decision. + * + * @param {import('./opentracing/span_context')} context + * @returns {{ randomValue: string, threshold: string } | undefined} + */ +function generateFields (context) { + const { priority } = context._sampling + const probabilityRate = getProbabilityRate(context) + if (priority === undefined || probabilityRate === undefined) return + + const thresholdValue = thresholdFor(probabilityRate) + let randomValue = randomValueFor(context._traceId.toBigInt()) + const kept = priority >= AUTO_KEEP + + if (kept && randomValue < thresholdValue) { + randomValue = thresholdValue + } else if (!kept && randomValue >= thresholdValue) { + randomValue = thresholdValue > 0n ? thresholdValue - 1n : 0n + } + + return { + randomValue: randomValue.toString(16).padStart(14, '0'), + threshold: formatThreshold(thresholdValue), + } +} + +/** + * Returns the probability rate recorded by a committed Datadog sampling decision. + * Sampling probes deliberately leave these rate fields unset. + * + * @param {import('./opentracing/span_context')} context + * @returns {number | undefined} + */ +function getProbabilityRate (context) { + if (context._sampling.isProbabilityDecision === false) return + return context._trace[SAMPLING_RULE_DECISION] ?? context._trace[SAMPLING_AGENT_DECISION] +} + +/** + * Adds a complete sub-field while the OTel member remains within its byte cap. + * + * @param {string[]} fields + * @param {string} field + * @param {number} byteLength + * @returns {number} + */ +function addField (fields, field, byteLength) { + const fieldLength = Buffer.byteLength(field) + (fields.length === 0 ? 0 : 1) + if (byteLength + fieldLength <= MAX_OTEL_VALUE_BYTES) { + fields.push(field) + return byteLength + fieldLength + } + return byteLength +} + +/** + * Parses and rebuilds the OTel tracestate member, preserving unknown sub-fields. + * + * @param {import('./opentracing/span_context')} context + * @param {string | undefined} member + * @returns {string | undefined} + */ +function buildOtelMember (context, member) { + if (member === undefined) { + if (context._sampling.isProbabilityDecision === false) return + const generated = generateFields(context) + if (!generated) return + return `rv:${generated.randomValue};th:${generated.threshold}` + } + + let randomValue + let threshold + const unknownFields = [] + let start = 0 + + while (start <= member.length) { + let end = member.indexOf(';', start) + if (end === -1) end = member.length + const field = member.slice(start, end) + if (field) { + const separator = field.indexOf(':') + const key = separator === -1 ? field : field.slice(0, separator) + const value = separator === -1 ? undefined : field.slice(separator + 1) + if (key === 'rv') { + randomValue = value + } else if (key === 'th') { + threshold = value + } else { + unknownFields.push(field) + } + } + if (end === member.length) break + start = end + 1 + } + + if (!validRandomValue.test(randomValue)) randomValue = undefined + if (!validThreshold.test(threshold)) threshold = undefined + + if (context._sampling.isProbabilityDecision === false) { + threshold = undefined + } else if (randomValue === undefined && threshold === undefined) { + const generated = generateFields(context) + if (generated) { + randomValue = generated.randomValue + threshold = generated.threshold + } + } + + const fields = [] + let byteLength = 0 + if (randomValue !== undefined) byteLength = addField(fields, `rv:${randomValue}`, byteLength) + if (threshold !== undefined) byteLength = addField(fields, `th:${threshold}`, byteLength) + for (const field of unknownFields) { + byteLength = addField(fields, field, byteLength) + } + + return fields.length === 0 ? undefined : fields.join(';') +} + +/** + * Updates the OTel tracestate member to represent the context's sampling decision. + * + * @param {import('./opentracing/span_context')} context + * @param {import('./opentracing/propagation/tracestate')} traceState + * @returns {void} + */ +function updateOtelTraceState (context, traceState) { + const otelMember = traceState.get('ot') + if (context._sampling.isProbabilityDecision === false) { + if (otelMember === undefined) return + } else if (getProbabilityRate(context) === undefined) { + return + } + + const rebuiltOtelMember = buildOtelMember(context, otelMember) + if (rebuiltOtelMember === undefined) { + traceState.delete('ot') + } else { + traceState.set('ot', rebuiltOtelMember) + } +} + +module.exports = { + buildOtelMember, + updateOtelTraceState, +} diff --git a/packages/dd-trace/src/priority_sampler.js b/packages/dd-trace/src/priority_sampler.js index 2728d98cf2c..9c485ccd289 100644 --- a/packages/dd-trace/src/priority_sampler.js +++ b/packages/dd-trace/src/priority_sampler.js @@ -26,8 +26,10 @@ const { SAMPLING_MECHANISM_AGENT, SAMPLING_MECHANISM_RULE, SAMPLING_MECHANISM_MANUAL, + SAMPLING_MECHANISM_APPSEC, SAMPLING_MECHANISM_REMOTE_USER, SAMPLING_MECHANISM_REMOTE_DYNAMIC, + SAMPLING_MECHANISM_AI_GUARD, SAMPLING_RULE_DECISION, SAMPLING_LIMIT_DECISION, SAMPLING_AGENT_DECISION, @@ -39,6 +41,18 @@ const DEFAULT_KEY = 'service:,env:' const defaultSampler = new Sampler(AUTO_KEEP) +/** + * Returns whether a product has already force-kept the trace. + * + * @param {import('./opentracing/span_context')} context + * @returns {boolean} + */ +function isProductForceKeep (context) { + const { priority, mechanism } = context._sampling + return priority === USER_KEEP && + (mechanism === SAMPLING_MECHANISM_APPSEC || mechanism === SAMPLING_MECHANISM_AI_GUARD) +} + /** * PrioritySampler is responsible for determining whether a span should be sampled * based on various rules, rate limits, and priorities. It supports manual and @@ -116,10 +130,10 @@ class PrioritySampler { const tag = this._getPriorityFromTags(context.getTags(), context) if (this.validate(tag)) { - context._sampling.priority = tag context._sampling.mechanism = SAMPLING_MECHANISM_MANUAL + this._recordDecision(context, tag) } else if (auto) { - context._sampling.priority = this._getPriorityFromAuto(root) + this.decideFromAuto(root) } else { return } @@ -127,6 +141,41 @@ class PrioritySampler { this.#addDecisionMaker(root) } + /** + * Applies one manual sampling tag without rereading the span's complete tag map. + * + * @param {DatadogSpan} span + * @param {string} key + * @param {unknown} value + * @returns {void} + */ + setPriorityFromTag (span, key, value) { + if (!span) return + + const context = this._getContext(span) + if (isProductForceKeep(context)) return + + const priority = this._getPriorityFromTag(key, value, context) + if (this.validate(priority)) this.setPriority(span, priority) + } + + /** + * Applies manual sampling tags from a supplied tag collection using their defined precedence. + * + * @param {DatadogSpan} span + * @param {Record} tags + * @returns {void} + */ + setPriorityFromTags (span, tags) { + if (!span) return + + const context = this._getContext(span) + if (isProductForceKeep(context)) return + + const priority = this._getPriorityFromTags(tags, context) + if (this.validate(priority)) this.setPriority(span, priority) + } + /** * Updates agent-provided sampling rates keyed by `service:,env:`. * @@ -183,14 +232,13 @@ class PrioritySampler { return // noop span } - context._sampling.priority = samplingPriority - const mechanism = product?.mechanism ?? SAMPLING_MECHANISM_MANUAL context._sampling.mechanism = mechanism + this._recordDecision(context, samplingPriority) log.trace(span, samplingPriority, mechanism) - this.#addDecisionMaker(root) + this.#addDecisionMaker(root, product === undefined) } /** @@ -205,6 +253,23 @@ class PrioritySampler { : /** @type {DatadogSpanContext} */ (span) } + /** + * Evaluates and records an automatic sampling decision. + * + * @param {DatadogSpan} span + * @returns {SamplingPriority} + */ + decideFromAuto (span) { + const context = this._getContext(span) + const rule = this.#findRule(span) + const priority = rule + ? this.#getPriorityByRule(context, rule, true) + : this.#getPriorityByAgent(context, true) + + context._sampling.priority = priority + return priority + } + /** * Computes priority using rules and agent rates when no manual tag is present. * @@ -216,27 +281,25 @@ class PrioritySampler { const rule = this.#findRule(span) return rule - ? this.#getPriorityByRule(context, rule) - : this.#getPriorityByAgent(context) + ? this.#getPriorityByRule(context, rule, false) + : this.#getPriorityByAgent(context, false) } /** - * Computes priority from manual sampling tags if present. - * Included for compatibility with {@link import('./standalone/tracesource_priority_sampler')._getPriorityFromTags} + * Converts one sampling tag into a user priority. * - * @param {Record} tags + * @param {string} key + * @param {unknown} value * @param {DatadogSpanContext} _context * @returns {SamplingPriority|undefined} */ - _getPriorityFromTags (tags, _context) { - if (Object.hasOwn(tags, MANUAL_KEEP) && tags[MANUAL_KEEP] !== false) { + _getPriorityFromTag (key, value, _context) { + if (key === MANUAL_KEEP && value !== false) { return USER_KEEP - } else if (Object.hasOwn(tags, MANUAL_DROP) && tags[MANUAL_DROP] !== false) { + } else if (key === MANUAL_DROP && value !== false) { return USER_REJECT - } - const rawPriority = tags[SAMPLING_PRIORITY] - if (rawPriority !== undefined) { - const priority = Math.trunc(rawPriority) + } else if (key === SAMPLING_PRIORITY && (typeof value === 'number' || typeof value === 'string')) { + const priority = Math.trunc(/** @type {number} */ (value)) if (priority === 1 || priority === 2) { return USER_KEEP @@ -246,15 +309,58 @@ class PrioritySampler { } } + /** + * Computes priority from manual sampling tags if present. + * Calls the overridable single-tag conversion so specialized samplers retain their filtering rules. + * + * @param {Record} tags + * @param {DatadogSpanContext} _context + * @returns {SamplingPriority|undefined} + */ + _getPriorityFromTags (tags, _context) { + if (!Object.hasOwn(tags, MANUAL_KEEP) && !Object.hasOwn(tags, MANUAL_DROP)) { + const priority = tags[SAMPLING_PRIORITY] + if (priority !== undefined) return this._getPriorityFromTag(SAMPLING_PRIORITY, priority, _context) + return + } + + return this.#getPriorityFromManualTags(tags, _context) + } + + /** + * Computes priority when at least one manual keep/drop tag is present. + * Value checks cover the common case; own-property checks preserve undefined-valued manual tags. + * + * @param {Record} tags + * @param {DatadogSpanContext} context + * @returns {SamplingPriority|undefined} + */ + #getPriorityFromManualTags (tags, context) { + const manualKeep = tags[MANUAL_KEEP] + if (manualKeep !== undefined || Object.hasOwn(tags, MANUAL_KEEP)) { + const priority = this._getPriorityFromTag(MANUAL_KEEP, manualKeep, context) + if (priority !== undefined) return priority + } + + const manualDrop = tags[MANUAL_DROP] + if (manualDrop !== undefined || Object.hasOwn(tags, MANUAL_DROP)) { + const priority = this._getPriorityFromTag(MANUAL_DROP, manualDrop, context) + if (priority !== undefined) return priority + } + const priority = tags[SAMPLING_PRIORITY] + if (priority !== undefined) return this._getPriorityFromTag(SAMPLING_PRIORITY, priority, context) + } + /** * Applies a matching rule and rate limit to compute the sampling priority. * * @param {DatadogSpanContext} context * @param {import('./sampling_rule')} rule + * @param {boolean} recordDecision * @returns {SamplingPriority} */ - #getPriorityByRule (context, rule) { - context._trace[SAMPLING_RULE_DECISION] = rule.sampleRate + #getPriorityByRule (context, rule, recordDecision) { + if (recordDecision) context._trace[SAMPLING_RULE_DECISION] = rule.sampleRate context._trace.tags[SAMPLING_KNUTH_RATE] = formatKnuthRate(rule.sampleRate) context._sampling.mechanism = SAMPLING_MECHANISM_RULE if (rule.provenance === 'customer') { @@ -267,9 +373,18 @@ class PrioritySampler { context._sampling.discard = true } - return rule.sample(context) && this._isSampledByRateLimit(context) - ? USER_KEEP - : USER_REJECT + const sampled = rule.sample(context, true) + if (sampled !== true) { + if (recordDecision && sampled === undefined) this._recordDecisionMetadata(context) + return USER_REJECT + } + + if (!this._isSampledByRateLimit(context)) { + if (recordDecision) this._recordDecisionMetadata(context) + return USER_REJECT + } + + return USER_KEEP } /** @@ -292,15 +407,16 @@ class PrioritySampler { * Computes priority using agent-provided sampling rates. * * @param {DatadogSpanContext} context + * @param {boolean} recordDecision * @returns {SamplingPriority} */ - #getPriorityByAgent (context) { + #getPriorityByAgent (context, recordDecision) { const key = `service:${context.getTag(SERVICE_NAME)},env:${this._env}` // TODO: Change underscored properties to private ones. const sampler = this._samplers[key] || this._samplers[DEFAULT_KEY] const rate = sampler.rate() - context._trace[SAMPLING_AGENT_DECISION] = rate + if (recordDecision) context._trace[SAMPLING_AGENT_DECISION] = rate if (sampler === defaultSampler) { context._sampling.mechanism = SAMPLING_MECHANISM_DEFAULT @@ -312,20 +428,44 @@ class PrioritySampler { return sampler.isSampled(context) ? AUTO_KEEP : AUTO_REJECT } + /** + * Records that a sampling decision does not represent a probability. + * + * @param {DatadogSpanContext} context + * @returns {void} + */ + _recordDecisionMetadata (context) { + context._sampling.isProbabilityDecision = false + } + + /** + * Records a local non-probability sampling decision. + * + * @param {DatadogSpanContext} context + * @param {SamplingPriority} priority + * @returns {SamplingPriority} + */ + _recordDecision (context, priority) { + context._sampling.priority = priority + this._recordDecisionMetadata(context) + return priority + } + /** * Tags the trace with a decision maker when priority is keep, or removes it otherwise. * * @param {DatadogSpan} span + * @param {boolean} [overwrite] * @returns {void} */ - #addDecisionMaker (span) { + #addDecisionMaker (span, overwrite = false) { const context = span.context() const trace = context._trace const priority = context._sampling.priority const mechanism = context._sampling.mechanism if (priority >= AUTO_KEEP) { - if (!trace.tags[DECISION_MAKER_KEY]) { + if (overwrite || !trace.tags[DECISION_MAKER_KEY]) { trace.tags[DECISION_MAKER_KEY] = `-${mechanism}` } } else if (trace.tags[DECISION_MAKER_KEY] !== undefined) { diff --git a/packages/dd-trace/src/sampler.js b/packages/dd-trace/src/sampler.js index df9eadb1dec..14fe1be6b87 100644 --- a/packages/dd-trace/src/sampler.js +++ b/packages/dd-trace/src/sampler.js @@ -5,10 +5,7 @@ // as it is cast into a float64 when computing the threshold const MAX_TRACE_ID = 2 ** 64 - 1 -const UINT64_MODULO = 2n ** 64n - -// Knuth's factor for the sampling algorithm -const SAMPLING_KNUTH_FACTOR = 1_111_111_111_111_111_111n +const knuthHash = require('./knuth-hash') /** * `Sampler` determines whether or not to sample a trace/span based on the trace ID. @@ -56,7 +53,7 @@ class Sampler { span = typeof span.context === 'function' ? span.context() : span - return (span._traceId.toBigInt() * SAMPLING_KNUTH_FACTOR) % UINT64_MODULO <= this.#threshold + return knuthHash(span._traceId.toBigInt()) <= this.#threshold } } diff --git a/packages/dd-trace/src/sampling_rule.js b/packages/dd-trace/src/sampling_rule.js index 1d733086c8d..92180491afd 100644 --- a/packages/dd-trace/src/sampling_rule.js +++ b/packages/dd-trace/src/sampling_rule.js @@ -280,17 +280,13 @@ class SamplingRule { * Determines whether a span should be sampled based on the configured sampling rule. * * @param {DatadogSpan|DatadogSpanContext} span - The span or span context to evaluate. - * @returns {boolean} `true` if the span should be sampled, otherwise `false`. + * @param {boolean} [distinguishRateLimit] Return `undefined` instead of `false` for a rate-limit rejection. + * @returns {boolean|undefined} `true` if the span should be sampled, otherwise `false` or `undefined`. */ - sample (span) { - if (!this._sampler.isSampled(span)) { - return false - } - - if (this._limiter) { - return this._limiter.isAllowed() - } + sample (span, distinguishRateLimit = false) { + if (!this._sampler.isSampled(span)) return false + if (this._limiter && !this._limiter.isAllowed()) return distinguishRateLimit ? undefined : false return true } } diff --git a/packages/dd-trace/src/standalone/index.js b/packages/dd-trace/src/standalone/index.js index 4d85c6f942b..ec7c51f8907 100644 --- a/packages/dd-trace/src/standalone/index.js +++ b/packages/dd-trace/src/standalone/index.js @@ -28,6 +28,7 @@ function onSpanExtract ({ spanContext = {} }) { spanContext._sampling.priority = undefined } else if (spanContext._sampling.priority !== USER_KEEP) { spanContext._sampling.priority = USER_KEEP + spanContext._sampling.isProbabilityDecision = false } } diff --git a/packages/dd-trace/src/standalone/tracesource_priority_sampler.js b/packages/dd-trace/src/standalone/tracesource_priority_sampler.js index 7d822e5630b..f063d9e1e15 100644 --- a/packages/dd-trace/src/standalone/tracesource_priority_sampler.js +++ b/packages/dd-trace/src/standalone/tracesource_priority_sampler.js @@ -21,9 +21,9 @@ class TraceSourcePrioritySampler extends PrioritySampler { * @override * @returns {import('../priority_sampler').SamplingPriority|undefined} */ - _getPriorityFromTags (tags, context) { - if (Object.hasOwn(tags, MANUAL_KEEP) && - tags[MANUAL_KEEP] !== false && + _getPriorityFromTag (key, value, context) { + if (key === MANUAL_KEEP && + value !== false && hasTraceSourcePropagationTag(context._trace.tags) ) { return USER_KEEP @@ -32,11 +32,15 @@ class TraceSourcePrioritySampler extends PrioritySampler { /** * @override + * @param {import('../opentracing/span')} span + * @param {boolean} [recordDecision] + * @returns {import('../priority_sampler').SamplingPriority} */ - _getPriorityFromAuto (span) { + _getPriorityFromAuto (span, recordDecision = false) { const context = this._getContext(span) context._sampling.mechanism = SAMPLING_MECHANISM_DEFAULT + if (recordDecision) this._recordDecisionMetadata(context) if (hasTraceSourcePropagationTag(context._trace.tags)) { return USER_KEEP @@ -45,6 +49,20 @@ class TraceSourcePrioritySampler extends PrioritySampler { return this._isSampledByRateLimit(context) ? AUTO_KEEP : AUTO_REJECT } + /** + * Evaluates and records the standalone automatic sampling decision. + * + * @override + * @param {import('../opentracing/span')} span + * @returns {import('../priority_sampler').SamplingPriority} + */ + decideFromAuto (span) { + const context = this._getContext(span) + const priority = this._getPriorityFromAuto(span, true) + context._sampling.priority = priority + return priority + } + /** * @override */ diff --git a/packages/dd-trace/test/opentelemetry/tracer_provider.spec.js b/packages/dd-trace/test/opentelemetry/tracer_provider.spec.js index 8c758f6ff7e..68aac345b6e 100644 --- a/packages/dd-trace/test/opentelemetry/tracer_provider.spec.js +++ b/packages/dd-trace/test/opentelemetry/tracer_provider.spec.js @@ -4,7 +4,7 @@ const assert = require('node:assert/strict') const { describe, it } = require('mocha') const sinon = require('sinon') -const { trace } = require('@opentelemetry/api') +const { context, propagation, trace } = require('@opentelemetry/api') require('../setup/core') const TracerProvider = require('../../src/opentelemetry/tracer_provider') @@ -28,6 +28,35 @@ describe('OTel TracerProvider', () => { assert.strictEqual(tracer, provider.getTracer()) }) + it('should inject OTel probability sampling state with the default propagator', () => { + const provider = new TracerProvider() + provider.register() + const span = provider.getTracer().startSpan('test') + const carrier = {} + + propagation.inject(trace.setSpan(context.active(), span), carrier) + + assert.match(carrier.tracestate, /^ot=rv:[0-9a-f]{14};th:0$/) + span.end() + }) + + it('should preserve inherited OTel sampling state with the default propagator', () => { + const provider = new TracerProvider() + provider.register() + const incomingCarrier = { + traceparent: '00-0123456789abcdef0123456789abcdef-0123456789abcdef-01', + tracestate: 'ot=rv:123456789abcde;th:8,vendor=value', + } + const parentContext = propagation.extract(context.active(), incomingCarrier) + const span = provider.getTracer().startSpan('test', {}, parentContext) + const outgoingCarrier = {} + + propagation.inject(trace.setSpan(parentContext, span), outgoingCarrier) + + assert.strictEqual(outgoingCarrier.tracestate, incomingCarrier.tracestate) + span.end() + }) + it('should get unique tracers by name and version key', () => { const provider = new TracerProvider() const tracer = provider.getTracer('a', '1') diff --git a/packages/dd-trace/test/opentracing/propagation/otel-sampling.spec.js b/packages/dd-trace/test/opentracing/propagation/otel-sampling.spec.js new file mode 100644 index 00000000000..621ffdb2977 --- /dev/null +++ b/packages/dd-trace/test/opentracing/propagation/otel-sampling.spec.js @@ -0,0 +1,379 @@ +'use strict' + +const assert = require('node:assert/strict') + +const { beforeEach, describe, it } = require('mocha') + +require('../../setup/core') +const { AUTO_KEEP, USER_KEEP, USER_REJECT } = require('../../../../../ext/priority') +const getConfig = require('../../../src/config') +const id = require('../../../src/id') +const Span = require('../../../src/opentracing/span') +const SpanContext = require('../../../src/opentracing/span_context') +const TextMapPropagator = require('../../../src/opentracing/propagation/text_map') +const TraceState = require('../../../src/opentracing/propagation/tracestate') +const PrioritySampler = require('../../../src/priority_sampler') +const { ASM } = require('../../../src/standalone/product') + +describe('OpenTelemetry consistent probability sampling propagation', () => { + let config + let propagator + + beforeEach(() => { + config = getConfig() + config.tracePropagationStyle = { + extract: ['tracecontext'], + inject: ['tracecontext'], + } + config.DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT = 'continue' + propagator = new TextMapPropagator(config) + }) + + describe('local probability decisions', () => { + const vectors = [ + [0.01, 'fd70a3d70a3d7', false], + [0.1, 'e6666666666668', true], + [0.2, 'ccccccccccccd', true], + [0.5, '8', true], + [0.99, '028f5c28f5c29', true], + ] + + for (const [sampleRate, expectedThreshold, expectedKept] of vectors) { + it(`emits the golden vector for rate ${sampleRate}`, () => { + const { carrier, context } = sampleAndInject({ traceId: '1', sampleRate }) + + assert.deepStrictEqual(parseOtel(carrier.tracestate), { + rv: 'f0948a54d43b8e', + th: expectedThreshold, + }) + assert.strictEqual(context._sampling.priority >= AUTO_KEEP, expectedKept) + }) + } + + it('emits the RFC worked example', () => { + const { carrier } = sampleAndInject({ + traceId: '18444899399302180863', + sampleRate: 0.1, + }) + + assert.deepStrictEqual(parseOtel(carrier.tracestate), { + rv: 'ef284ace7a91e1', + th: 'e6666666666668', + }) + }) + + it('adjusts rv up when 56-bit precision would reverse a keep', () => { + const { carrier, context } = sampleAndInject({ + traceId: '263811222310854400', + sampleRate: 0.1, + }) + + assert.deepStrictEqual(parseOtel(carrier.tracestate), { + rv: 'e6666666666668', + th: 'e6666666666668', + }) + assert.strictEqual(context._sampling.priority, USER_KEEP) + }) + + it('adjusts rv down when 56-bit precision would reverse a drop', () => { + const { carrier, context } = sampleAndInject({ + traceId: '5401449561355763072', + sampleRate: 0.05, + }) + + assert.deepStrictEqual(parseOtel(carrier.tracestate), { + rv: 'f333333333332f', + th: 'f333333333333', + }) + assert.strictEqual(context._sampling.priority, USER_REJECT) + }) + + it('emits a probability decision for the default agent rate', () => { + const { carrier, context } = sampleAndInject({ traceId: '1' }) + + assert.deepStrictEqual(parseOtel(carrier.tracestate), { + rv: 'f0948a54d43b8e', + th: '0', + }) + assert.strictEqual(context._sampling.priority, AUTO_KEEP) + }) + }) + + describe('inherited decisions', () => { + it('forwards inherited state without changing unrelated member order', () => { + const parent = extractParent({ + sampled: true, + tracestate: 'dd=s:2;t.dm:-3,congo=t61rcWkgMzE,' + + 'ot=th:e6666666666668;foo:bar;rv:ef284ace7a91e1,something=else', + }) + const { span, prioritySampler } = startSpan({ parent, sampleRate: 0 }) + const carrier = inject(span, prioritySampler) + const members = carrier.tracestate.split(',') + + assert.match(members[0], /^dd=/) + assert.deepStrictEqual(members.slice(1), [ + 'congo=t61rcWkgMzE', + 'ot=th:e6666666666668;foo:bar;rv:ef284ace7a91e1', + 'something=else', + ]) + assert.strictEqual(parseTracestate(carrier.tracestate).congo, 't61rcWkgMzE') + assert.strictEqual(span.context()._sampling.priority, USER_KEEP) + }) + + for (const sampled of [false, true]) { + it(`forwards a th-only ${sampled ? 'keep' : 'drop'} without fabricating rv`, () => { + const parent = extractParent({ sampled, tracestate: 'ot=th:e6666666666668' }) + const { span, prioritySampler } = startSpan({ parent, sampleRate: 0.5 }) + const carrier = inject(span, prioritySampler) + + assert.deepStrictEqual(parseOtel(carrier.tracestate), { th: 'e6666666666668' }) + }) + } + + for (const [source, sampleRate] of [['rule-based', 0.1], ['agent-based', undefined]]) { + it(`does not fabricate fields after probing ${source} sampling on a sampled trace without ot`, () => { + const parent = extractParent({ sampled: true, tracestate: 'dd=s:1' }) + const { span, prioritySampler } = startSpan({ parent, sampleRate }) + + prioritySampler.isSampled(span) + const carrier = inject(span, prioritySampler) + + assert.strictEqual(parseTracestate(carrier.tracestate).ot, undefined) + }) + } + + it('preserves an unknown-only ot member without fabricating sampling fields', () => { + const parent = extractParent({ sampled: true, tracestate: 'ot=foo:bar' }) + const { span, prioritySampler } = startSpan({ parent, sampleRate: 0.1 }) + const carrier = inject(span, prioritySampler) + + assert.deepStrictEqual(parseOtel(carrier.tracestate), { foo: 'bar' }) + }) + + it('forwards malformed inherited rv and th unchanged', () => { + const malformedBoth = extractParent({ + sampled: true, + tracestate: 'dd=s:1,ot=rv:not-hex;th:not-hex,congo=value', + }) + const first = startSpan({ parent: malformedBoth, sampleRate: 0.1 }) + const firstCarrier = inject(first.span, first.prioritySampler) + + assert.strictEqual(parseTracestate(firstCarrier.tracestate).ot, 'rv:not-hex;th:not-hex') + assert.strictEqual(parseTracestate(firstCarrier.tracestate).congo, 'value') + + const malformedThreshold = extractParent({ + sampled: true, + tracestate: 'ot=rv:1234567890abcd;th:not-hex', + }) + const second = startSpan({ parent: malformedThreshold, sampleRate: 0.1 }) + const secondCarrier = inject(second.span, second.prioritySampler) + + assert.strictEqual(parseTracestate(secondCarrier.tracestate).ot, 'rv:1234567890abcd;th:not-hex') + }) + }) + + describe('non-probability decisions', () => { + it('clears an inherited th and forwards rv when a manual tag overrides an upstream drop', () => { + const parent = extractParent({ + sampled: false, + traceId: '10', + tracestate: 'ot=rv:65cd67504a538e;th:e6666666666668', + }) + const { span, prioritySampler } = startSpan({ parent, sampleRate: 0.1 }) + + span.setTag('manual.keep', true) + const carrier = inject(span, prioritySampler) + + assert.deepStrictEqual(parseOtel(carrier.tracestate), { rv: '65cd67504a538e' }) + assert.strictEqual(span.context()._sampling.priority, USER_KEEP) + }) + + it('does not create ot fields for a local force-keep', () => { + const { span, prioritySampler } = startSpan({ traceId: '1', sampleRate: 0.1 }) + + prioritySampler.setPriority(span, USER_KEEP, ASM) + const carrier = inject(span, prioritySampler) + + assert.strictEqual(parseTracestate(carrier.tracestate).ot, undefined) + }) + + it('does not create ot fields when a limiter turns a probability keep into a drop', () => { + const { carrier, context } = sampleAndInject({ traceId: '1', sampleRate: 1, rateLimit: 0 }) + + assert.strictEqual(parseTracestate(carrier.tracestate).ot, undefined) + assert.strictEqual(context._sampling.priority, USER_REJECT) + }) + + it('keeps probability fields when the probability decision itself drops', () => { + const { carrier, context } = sampleAndInject({ traceId: '1', sampleRate: 0, rateLimit: 0 }) + + assert.deepStrictEqual(parseOtel(carrier.tracestate), { + rv: 'f0948a54d43b8e', + th: 'ffffffffffffff', + }) + assert.strictEqual(context._sampling.priority, USER_REJECT) + }) + }) + + describe('limits and extraction behavior', () => { + it('retains dd and ot as the first two of 32 members', () => { + const tracestate = Array.from({ length: 32 }, (_, index) => `v${index}=x`).join(',') + const { span, prioritySampler } = startSpan({ traceId: '1', sampleRate: 0.1 }) + span.context()._tracestate = TraceState.fromString(tracestate) + + const members = inject(span, prioritySampler).tracestate.split(',') + + assert.strictEqual(members.length, 32) + assert.match(members[0], /^dd=/) + assert.match(members[1], /^ot=/) + assert.strictEqual(members[31], 'v29=x') + assert.ok(!members.includes('v30=x')) + }) + + it('keeps complete OTel sub-fields within the 256-byte value cap', () => { + const oversized = 'future:' + 'x'.repeat(220) + const { span, prioritySampler } = startSpan({ traceId: '1', sampleRate: 0.1 }) + span.context()._tracestate = TraceState.fromString(`ot=${oversized};next:value`) + + const value = parseTracestate(inject(span, prioritySampler).tracestate).ot + + assert.ok(Buffer.byteLength(value) <= 256) + assert.deepStrictEqual(parseOtelValue(value), { + rv: 'f0948a54d43b8e', + th: 'e6666666666668', + next: 'value', + }) + }) + + for (const behavior of ['ignore', 'restart']) { + it(`ignores inbound ot fields and creates a new decision with ${behavior}`, () => { + config.DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT = behavior + const parent = extractParent({ + sampled: true, + tracestate: 'ot=rv:1234567890abcd;th:8', + }) + const { span, prioritySampler } = startSpan({ parent, sampleRate: 0.1 }) + const fields = parseOtel(inject(span, prioritySampler).tracestate) + + assert.strictEqual(fields.th, 'e6666666666668') + assert.notStrictEqual(fields.rv, '1234567890abcd') + }) + } + }) + + /** + * Starts a real span and priority sampler, optionally continuing a remote context. + * + * @param {object} options + * @param {string} [options.traceId] + * @param {number} [options.sampleRate] + * @param {number} [options.rateLimit] + * @param {SpanContext} [options.parent] + * @returns {{ span: Span, prioritySampler: PrioritySampler }} + */ + function startSpan ({ traceId = '1', sampleRate, rateLimit = -1, parent } = {}) { + const samplingConfig = { rateLimit } + if (sampleRate !== undefined) samplingConfig.sampleRate = sampleRate + const prioritySampler = new PrioritySampler('test', samplingConfig) + parent ??= new SpanContext({ + traceId: id(traceId, 10), + spanId: id('1', 10), + isRemote: false, + }) + const span = new Span( + { _config: config, _service: 'test' }, + { process () {} }, + prioritySampler, + { operationName: 'test', parent } + ) + span.context().setTag('service.name', 'test') + return { span, prioritySampler } + } + + /** + * Samples and injects a locally started trace. + * + * @param {object} options + * @param {string} options.traceId + * @param {number} [options.sampleRate] + * @param {number} [options.rateLimit] + * @returns {{ carrier: Record, context: SpanContext }} + */ + function sampleAndInject (options) { + const { span, prioritySampler } = startSpan(options) + return { + carrier: inject(span, prioritySampler), + context: span.context(), + } + } + + /** + * Applies priority sampling and injects tracecontext headers. + * + * @param {Span} span + * @param {PrioritySampler} prioritySampler + * @returns {Record} + */ + function inject (span, prioritySampler) { + prioritySampler.sample(span) + return propagator.inject(span.context(), {}) + } + + /** + * Extracts a remote W3C parent context. + * + * @param {object} options + * @param {boolean} options.sampled + * @param {string} options.tracestate + * @param {string} [options.traceId] + * @returns {SpanContext} + */ + function extractParent ({ sampled, tracestate, traceId = '18444899399302180863' }) { + const traceIdHex = BigInt(traceId).toString(16).padStart(32, '0') + return propagator.extract({ + traceparent: `00-${traceIdHex}-0000000000000001-${sampled ? '01' : '00'}`, + tracestate, + }) + } +}) + +/** + * Parses tracestate members by vendor. + * + * @param {string} tracestate + * @returns {Record} + */ +function parseTracestate (tracestate) { + const result = {} + for (const member of tracestate.split(',')) { + const separator = member.indexOf('=') + result[member.slice(0, separator)] = member.slice(separator + 1) + } + return result +} + +/** + * Parses an OTel member from tracestate. + * + * @param {string} tracestate + * @returns {Record} + */ +function parseOtel (tracestate) { + return parseOtelValue(parseTracestate(tracestate).ot) +} + +/** + * Parses OTel sub-fields. + * + * @param {string | undefined} value + * @returns {Record} + */ +function parseOtelValue (value) { + const result = {} + if (value === undefined) return result + for (const field of value.split(';')) { + const separator = field.indexOf(':') + if (separator !== -1) result[field.slice(0, separator)] = field.slice(separator + 1) + } + return result +} diff --git a/packages/dd-trace/test/opentracing/propagation/text_map.spec.js b/packages/dd-trace/test/opentracing/propagation/text_map.spec.js index d8bf15151e5..ffc52de4908 100644 --- a/packages/dd-trace/test/opentracing/propagation/text_map.spec.js +++ b/packages/dd-trace/test/opentracing/propagation/text_map.spec.js @@ -21,6 +21,7 @@ const { SAMPLING_MECHANISM_MANUAL } = require('../../../src/constants') // v5 spells single-header B3 propagation as `'b3 single header'`; v6+ reuses `'b3'` for it. const B3_SINGLE_STYLE = DD_MAJOR >= 6 ? 'b3' : 'b3 single header' +const B3_MULTI_STYLE = DD_MAJOR >= 6 ? 'b3multi' : 'b3' const injectCh = channel('dd-trace:span:inject') const extractCh = channel('dd-trace:span:extract') @@ -1508,6 +1509,130 @@ describe('TextMapPropagator', () => { assert.strictEqual(spanContext._tracestate.get('other'), 'bleh') }) + it('should propagate tracecontext tracestate when matching B3 headers take precedence', () => { + const traceId = '1111aaaa2222bbbb3333cccc4444dddd' + const spanId = '5555eeee6666ffff' + textMap = { + b3: `${traceId}-${spanId}-1`, + traceparent: `00-${traceId}-${spanId}-01`, + tracestate: 'ot=rv:123456789abcde;th:8', + } + config.tracePropagationStyle.extract = [B3_SINGLE_STYLE, 'tracecontext'] + config.tracePropagationStyle.inject = ['tracecontext'] + + const spanContext = propagator.extract(textMap) + const carrier = propagator.inject(spanContext, {}) + + assert.strictEqual(spanContext._tracestate.get('ot'), 'rv:123456789abcde;th:8') + assert.match(carrier.tracestate, /(?:^|,)ot=rv:123456789abcde;th:8(?:,|$)/) + }) + + it('should inherit a tracecontext drop when matching B3 headers omit a sampling decision', () => { + const traceId = '1111aaaa2222bbbb3333cccc4444dddd' + const spanId = '5555eeee6666ffff' + textMap = { + b3: `${traceId}-${spanId}`, + traceparent: `00-${traceId}-${spanId}-00`, + tracestate: 'ot=rv:00000000000000;th:8', + } + config.tracePropagationStyle.extract = [B3_SINGLE_STYLE, 'tracecontext'] + config.tracePropagationStyle.inject = ['tracecontext'] + + const spanContext = propagator.extract(textMap) + const carrier = propagator.inject(spanContext, {}) + + assert.strictEqual(spanContext._sampling.priority, AUTO_REJECT) + assert.match(carrier.traceparent, /-00$/) + assert.match(carrier.tracestate, /(?:^|,)ot=rv:00000000000000;th:8(?:,|$)/) + }) + + it('should inherit a tracecontext keep when matching Datadog headers omit a sampling decision', () => { + textMap = { + 'x-datadog-trace-id': '123', + 'x-datadog-parent-id': '456', + traceparent: '00-0000000000000000000000000000007b-00000000000001c8-01', + tracestate: 'ot=rv:ffffffffffffff;th:8', + } + config.tracePropagationStyle.extract = ['datadog', 'tracecontext'] + config.tracePropagationStyle.inject = ['tracecontext'] + + const spanContext = propagator.extract(textMap) + const carrier = propagator.inject(spanContext, {}) + + assert.strictEqual(spanContext._sampling.priority, AUTO_KEEP) + assert.match(carrier.traceparent, /-01$/) + assert.match(carrier.tracestate, /(?:^|,)ot=rv:ffffffffffffff;th:8(?:,|$)/) + }) + + for (const [style, b3Headers] of [ + [B3_SINGLE_STYLE, { b3: '1111aaaa2222bbbb3333cccc4444dddd-5555eeee6666ffff-d' }], + [B3_MULTI_STYLE, { + 'x-b3-traceid': '1111aaaa2222bbbb3333cccc4444dddd', + 'x-b3-spanid': '5555eeee6666ffff', + 'x-b3-flags': '1', + }], + ]) { + it(`should clear the W3C threshold when a selected ${style} debug decision agrees with tracecontext`, () => { + const traceId = '1111aaaa2222bbbb3333cccc4444dddd' + const spanId = '5555eeee6666ffff' + textMap = { + ...b3Headers, + traceparent: `00-${traceId}-${spanId}-01`, + tracestate: 'ot=rv:ffffffffffffff;th:8;vendor:value', + } + config.tracePropagationStyle.extract = [style, 'tracecontext'] + config.tracePropagationStyle.inject = ['tracecontext'] + + const spanContext = propagator.extract(textMap) + const carrier = propagator.inject(spanContext, {}) + + assert.strictEqual(spanContext._sampling.priority, USER_KEEP) + assert.strictEqual(spanContext._sampling.isProbabilityDecision, false) + assert.match(carrier.traceparent, /-01$/) + assert.match(carrier.tracestate, /(?:^|,)ot=rv:ffffffffffffff;vendor:value(?:,|$)/) + }) + } + + it('should clear the W3C threshold when a selected B3 keep conflicts with tracecontext', () => { + const traceId = '1111aaaa2222bbbb3333cccc4444dddd' + const spanId = '5555eeee6666ffff' + textMap = { + b3: `${traceId}-${spanId}-1`, + traceparent: `00-${traceId}-${spanId}-00`, + tracestate: 'ot=rv:00000000000000;th:8;vendor:value', + } + config.tracePropagationStyle.extract = [B3_SINGLE_STYLE, 'tracecontext'] + config.tracePropagationStyle.inject = ['tracecontext'] + + const spanContext = propagator.extract(textMap) + const carrier = propagator.inject(spanContext, {}) + + assert.strictEqual(spanContext._sampling.isProbabilityDecision, false) + assert.match(carrier.traceparent, /-01$/) + assert.match(carrier.tracestate, /(?:^|,)ot=rv:00000000000000;vendor:value(?:,|$)/) + }) + + it('should clear the W3C sampling state when a selected Datadog drop conflicts with tracecontext', () => { + textMap = { + 'x-datadog-trace-id': '123', + 'x-datadog-parent-id': '456', + 'x-datadog-sampling-priority': '-1', + traceparent: '00-0000000000000000000000000000007b-00000000000001c8-01', + tracestate: 'other=bleh,dd=t.dm:-3,ot=rv:ffffffffffffff;th:8;vendor:value', + } + config.tracePropagationStyle.extract = ['datadog', 'tracecontext'] + config.tracePropagationStyle.inject = ['tracecontext'] + + const spanContext = propagator.extract(textMap) + const carrier = propagator.inject(spanContext, {}) + + assert.strictEqual(spanContext._sampling.isProbabilityDecision, false) + assert.match(carrier.traceparent, /-00$/) + assert.match(carrier.tracestate, /(?:^|,)ot=rv:ffffffffffffff;vendor:value(?:,|$)/) + assert.match(carrier.tracestate, /(?:^|,)dd=s:-1(?:,|$)/) + assert.doesNotMatch(carrier.tracestate, /t\.dm:/) + }) + it('should read tracecontext once while resolving multiple propagation styles', () => { for (const extract of [['datadog', 'tracecontext'], ['tracecontext', 'datadog']]) { let reads = 0 diff --git a/packages/dd-trace/test/opentracing/propagation/tracestate.spec.js b/packages/dd-trace/test/opentracing/propagation/tracestate.spec.js index 2dc1a9b96df..0fda828c499 100644 --- a/packages/dd-trace/test/opentracing/propagation/tracestate.spec.js +++ b/packages/dd-trace/test/opentracing/propagation/tracestate.spec.js @@ -97,6 +97,30 @@ describe('TraceState', () => { assert.strictEqual(ts.size, 32) }) + it('should keep the 32 leftmost list-members after updates', () => { + const header = Array.from({ length: 32 }, (_, index) => `k${index}=v${index}`).join(',') + const ts = TraceState.fromString(header) + ts.set('ot', 'rv:f0948a54d43b8e;th:8') + ts.set('dd', 's:1') + + const members = ts.toString().split(',') + assert.strictEqual(members.length, 32) + assert.deepStrictEqual(members.slice(0, 3), ['dd=s:1', 'ot=rv:f0948a54d43b8e;th:8', 'k0=v0']) + assert.strictEqual(members[31], 'k29=v29') + }) + + it('should keep a tracestate at the 512-byte cap', () => { + const ts = TraceState.fromString(`a=${'x'.repeat(510)}`) + + assert.strictEqual(Buffer.byteLength(ts.toString()), 512) + }) + + it('should drop the first member beyond the 512-byte cap', () => { + const ts = TraceState.fromString(`a=${'x'.repeat(511)}`) + + assert.strictEqual(ts.toString(), '') + }) + it('should accept internal spaces but drop tabs in tracestate values per W3C Trace Context §3.3.1.3.2', () => { const ts = TraceState.fromString('a=hello world,b=bye\tworld,c=ok') assert.strictEqual(ts.toString(), 'a=hello world,c=ok') diff --git a/packages/dd-trace/test/opentracing/span.spec.js b/packages/dd-trace/test/opentracing/span.spec.js index f767eb6a470..e66603a0eb4 100644 --- a/packages/dd-trace/test/opentracing/span.spec.js +++ b/packages/dd-trace/test/opentracing/span.spec.js @@ -10,7 +10,7 @@ const proxyquire = require('proxyquire') const { assertObjectContains } = require('../../../../integration-tests/helpers') require('../setup/core') -const { MANUAL_KEEP } = require('../../../../ext/tags') +const { MANUAL_DROP, MANUAL_KEEP } = require('../../../../ext/tags') const { DD_MAJOR } = require('../../../../version') const getConfig = require('../../src/config') const TextMapPropagator = require('../../src/opentracing/propagation/text_map') @@ -46,6 +46,8 @@ describe('Span', () => { prioritySampler = { sample: sinon.stub(), + setPriorityFromTag: sinon.stub(), + setPriorityFromTags: sinon.stub(), } tagger = { @@ -594,7 +596,7 @@ describe('Span', () => { span.setTag(MANUAL_KEEP, true) assert.strictEqual(span.context().getTag(MANUAL_KEEP), true) - sinon.assert.calledWith(prioritySampler.sample, span, false) + sinon.assert.calledOnceWithExactly(prioritySampler.setPriorityFromTag, span, MANUAL_KEEP, true) }) it('should be published via dd-trace:span:tags:update channel', () => { @@ -637,13 +639,34 @@ describe('Span', () => { sinon.assert.notCalled(prioritySampler.sample) }) - const legacyAddTagsShape = DD_MAJOR < 6 ? it : it.skip - legacyAddTagsShape('still accepts string and array inputs via tagger on v5', () => { - span.addTags('foo:bar') - span.addTags([{ baz: 'qux' }]) + it('only reapplies sampling tags parsed from the current legacy v5 input', () => { + const legacyTagger = { add: sinon.spy(require('../../src/tagger').add) } + const LegacySpan = proxyquire('../../src/opentracing/span', { + perf_hooks: { performance: { now } }, + '../id': sinon.stub().returns('789'), + '../log': log, + '../tagger': legacyTagger, + '../../../../version': { DD_MAJOR: 5 }, + }) + const legacySpan = new LegacySpan(tracer, processor, prioritySampler, { operationName: 'operation' }) + + legacySpan.addTags(`${MANUAL_KEEP}:true`) + + assert.strictEqual(legacySpan.context().getTag(MANUAL_KEEP), 'true') + sinon.assert.calledOnceWithExactly( + prioritySampler.setPriorityFromTags, + legacySpan, + { [MANUAL_KEEP]: 'true' } + ) + + prioritySampler.setPriorityFromTags.resetHistory() + legacySpan.setTag(MANUAL_DROP, true) + legacySpan.addTags('foo:bar') + legacySpan.addTags([{ baz: 'qux' }]) - sinon.assert.calledWith(tagger.add, span.context().getTags(), 'foo:bar') - sinon.assert.calledWith(tagger.add, span.context().getTags(), [{ baz: 'qux' }]) + assert.strictEqual(legacySpan.context().getTag('foo'), 'bar') + assert.strictEqual(legacySpan.context().getTag('baz'), 'qux') + sinon.assert.notCalled(prioritySampler.setPriorityFromTags) }) const v6AddTagsShape = DD_MAJOR >= 6 ? it : it.skip @@ -658,10 +681,11 @@ describe('Span', () => { }) it('should sample based on manual sampling tags', () => { - span.addTags({ [MANUAL_KEEP]: true }) + const tags = { [MANUAL_KEEP]: true } + span.addTags(tags) assert.strictEqual(span.context().getTag(MANUAL_KEEP), true) - sinon.assert.calledWith(prioritySampler.sample, span, false) + sinon.assert.calledOnceWithExactly(prioritySampler.setPriorityFromTags, span, tags) }) it('should be published via dd-trace:span:tags:update channel', () => { diff --git a/packages/dd-trace/test/priority_sampler.spec.js b/packages/dd-trace/test/priority_sampler.spec.js index ca6f09c5cc4..e9b70eb0695 100644 --- a/packages/dd-trace/test/priority_sampler.spec.js +++ b/packages/dd-trace/test/priority_sampler.spec.js @@ -18,9 +18,10 @@ const { SAMPLING_MECHANISM_REMOTE_DYNAMIC, DECISION_MAKER_KEY, SAMPLING_MECHANISM_APPSEC, + SAMPLING_MECHANISM_AI_GUARD, SAMPLING_KNUTH_RATE, } = require('../src/constants') -const { ASM } = require('../src/standalone/product') +const { AI_GUARD, ASM } = require('../src/standalone/product') const SERVICE_NAME = ext.tags.SERVICE_NAME const SAMPLING_PRIORITY = ext.tags.SAMPLING_PRIORITY @@ -111,11 +112,28 @@ describe('PrioritySampler', () => { describe('isSampled', () => { it('should sample by default', () => { assert.strictEqual(prioritySampler.isSampled(span), true) + assert.strictEqual(context._trace['_dd.agent_psr'], undefined) + }) + + it('should not overwrite a committed manual decision while evaluating', () => { + prioritySampler.setPriority(span, USER_REJECT) + + assert.strictEqual(prioritySampler.isSampled(span), true) + assert.strictEqual(context._sampling.priority, USER_REJECT) + assert.strictEqual(context._sampling.probabilityRate, undefined) + assert.strictEqual(context._sampling.isProbabilityDecision, false) }) it('should accept a span context', () => { assert.strictEqual(prioritySampler.isSampled(context), true) }) + + it('should not record a rule decision while evaluating', () => { + prioritySampler = new PrioritySampler('test', { sampleRate: 0.5 }) + + assert.strictEqual(prioritySampler.isSampled(span), true) + assert.strictEqual(context._trace['_dd.rule_psr'], undefined) + }) }) describe('sample', () => { @@ -564,6 +582,70 @@ describe('PrioritySampler', () => { }) }) + describe('setPriorityFromTag', () => { + it('should let a manual sampling tag override an automatic decision', () => { + prioritySampler.sample(span) + assert.strictEqual(context._trace.tags[DECISION_MAKER_KEY], '-0') + + prioritySampler.setPriorityFromTag(span, SAMPLING_PRIORITY, `${USER_KEEP}`) + + assert.strictEqual(context._sampling.priority, USER_KEEP) + assert.strictEqual(context._sampling.mechanism, SAMPLING_MECHANISM_MANUAL) + assert.strictEqual(context._sampling.isProbabilityDecision, false) + assert.strictEqual(context._trace.tags[DECISION_MAKER_KEY], '-4') + }) + + it('should ignore a disabled manual sampling tag', () => { + prioritySampler.setPriorityFromTag(span, MANUAL_KEEP, false) + + assert.strictEqual(context._sampling.priority, undefined) + }) + + it('should ignore invalid sampling priority values after an automatic decision', () => { + prioritySampler.sample(span) + + for (const value of [1n, Symbol('priority')]) { + prioritySampler.setPriorityFromTag(span, SAMPLING_PRIORITY, value) + } + + assert.strictEqual(context._sampling.priority, AUTO_KEEP) + }) + + it('should not let a manual tag override an AppSec force-keep', () => { + prioritySampler.setPriority(span, USER_KEEP, ASM) + + prioritySampler.setPriorityFromTag(span, MANUAL_DROP, true) + + assert.strictEqual(context._sampling.priority, USER_KEEP) + assert.strictEqual(context._sampling.mechanism, SAMPLING_MECHANISM_APPSEC) + assert.strictEqual(context._trace.tags[DECISION_MAKER_KEY], '-5') + }) + }) + + describe('setPriorityFromTags', () => { + it('should apply precedence within the supplied sampling tags', () => { + prioritySampler.setPriorityFromTags(span, { + [MANUAL_KEEP]: false, + [MANUAL_DROP]: true, + [SAMPLING_PRIORITY]: USER_KEEP, + }) + + assert.strictEqual(context._sampling.priority, USER_REJECT) + assert.strictEqual(context._sampling.mechanism, SAMPLING_MECHANISM_MANUAL) + assert.strictEqual(context._sampling.isProbabilityDecision, false) + }) + + it('should not let manual tags override an AI Guard force-keep', () => { + prioritySampler.setPriority(span, USER_KEEP, AI_GUARD) + + prioritySampler.setPriorityFromTags(span, { [MANUAL_DROP]: true }) + + assert.strictEqual(context._sampling.priority, USER_KEEP) + assert.strictEqual(context._sampling.mechanism, SAMPLING_MECHANISM_AI_GUARD) + assert.strictEqual(context._trace.tags[DECISION_MAKER_KEY], '-13') + }) + }) + describe('update', () => { let rootSpan let rootContext diff --git a/packages/dd-trace/test/sampling_rule.spec.js b/packages/dd-trace/test/sampling_rule.spec.js index 3f765b0a10b..37068167aa9 100644 --- a/packages/dd-trace/test/sampling_rule.spec.js +++ b/packages/dd-trace/test/sampling_rule.spec.js @@ -594,6 +594,15 @@ describe('sampling rule', () => { assert.strictEqual(rule.sample(spanContext), false) }) + it('should distinguish probability and rate-limit rejections', () => { + const probabilityRule = new SamplingRule({ sampleRate: 0 }) + const limitedRule = new SamplingRule({ sampleRate: 1, maxPerSecond: 0 }) + const spanContext = new SpanContext({ traceId: id('2986627970102095326', 10) }) + + assert.strictEqual(probabilityRule.sample(spanContext, true), false) + assert.strictEqual(limitedRule.sample(spanContext, true), undefined) + }) + it('should allow unlimited rate limits', () => { rule = new SamplingRule({ service: 'test', diff --git a/packages/dd-trace/test/standalone/index.spec.js b/packages/dd-trace/test/standalone/index.spec.js index b78e803ef59..06e4d751c86 100644 --- a/packages/dd-trace/test/standalone/index.spec.js +++ b/packages/dd-trace/test/standalone/index.spec.js @@ -160,20 +160,28 @@ describe('Disabled APM Tracing or Standalone', () => { assert.strictEqual(spanContext._trace.tags[DECISION_MAKER_KEY], '-5') }) - it('should set USER_KEEP priority if _dd.p.ts=02 is present', () => { + it('should replace an extracted probability drop with a non-probability keep', () => { standalone.configure(config) const carrier = { - 'x-datadog-trace-id': '123123', - 'x-datadog-parent-id': '345345', - 'x-datadog-sampling-priority': '1', + 'x-datadog-trace-id': '123', + 'x-datadog-parent-id': '456', + 'x-datadog-sampling-priority': '-1', 'x-datadog-tags': '_dd.p.ts=02', + traceparent: '00-0000000000000000000000000000007b-00000000000001c8-00', + tracestate: 'ot=rv:123456789abcde;th:8', } const propagator = new TextMapPropagator(config) const spanContext = propagator.extract(carrier) assert.strictEqual(spanContext._sampling.priority, USER_KEEP) + assert.strictEqual(spanContext._sampling.probabilityRate, undefined) + assert.strictEqual(spanContext._sampling.isProbabilityDecision, false) + + const injected = propagator.inject(spanContext, {}) + assert.match(injected.traceparent, /-01$/) + assert.match(injected.tracestate, /(?:^|,)ot=rv:123456789abcde(?:,|$)/) }) it('should keep priority if apm tracing is enabled', () => { diff --git a/packages/dd-trace/test/standalone/tracesource_priority_sampler.spec.js b/packages/dd-trace/test/standalone/tracesource_priority_sampler.spec.js index 54a1e157b02..ea26941935b 100644 --- a/packages/dd-trace/test/standalone/tracesource_priority_sampler.spec.js +++ b/packages/dd-trace/test/standalone/tracesource_priority_sampler.spec.js @@ -11,7 +11,7 @@ const { USER_KEEP, AUTO_KEEP } = require('../../../../ext/priority') const getConfig = require('../../src/config') const DatadogSpan = require('../../src/opentracing/span') const TraceSourcePrioritySampler = require('../../src/standalone/tracesource_priority_sampler') -const { TRACE_SOURCE_PROPAGATION_KEY } = require('../../src/constants') +const { SAMPLING_MECHANISM_DEFAULT, TRACE_SOURCE_PROPAGATION_KEY } = require('../../src/constants') const { ASM } = require('../../src/standalone/product') describe('Disabled APM Tracing or Standalone - TraceSourcePrioritySampler', () => { @@ -72,6 +72,8 @@ describe('Disabled APM Tracing or Standalone - TraceSourcePrioritySampler', () = context._trace.tags[TRACE_SOURCE_PROPAGATION_KEY] = '02' assert.strictEqual(prioritySampler._getPriorityFromAuto(span), USER_KEEP) + assert.strictEqual(context._sampling.priority, undefined) + assert.strictEqual(context._sampling.mechanism, SAMPLING_MECHANISM_DEFAULT) }) it('should use rate limiter if it does not contain _dd.p.ts tag', () => { @@ -82,6 +84,21 @@ describe('Disabled APM Tracing or Standalone - TraceSourcePrioritySampler', () = sinon.stub(prioritySampler, '_isSampledByRateLimit').returns(true) assert.strictEqual(prioritySampler._getPriorityFromAuto(span), AUTO_KEEP) + assert.strictEqual(context._sampling.priority, undefined) + assert.strictEqual(context._sampling.mechanism, SAMPLING_MECHANISM_DEFAULT) + }) + + it('should record a committed automatic decision', () => { + const span = { + _trace: {}, + } + + context._trace.tags[TRACE_SOURCE_PROPAGATION_KEY] = '02' + + assert.strictEqual(prioritySampler.decideFromAuto(span), USER_KEEP) + assert.strictEqual(context._sampling.priority, USER_KEEP) + assert.strictEqual(context._sampling.mechanism, SAMPLING_MECHANISM_DEFAULT) + assert.strictEqual(context._sampling.isProbabilityDecision, false) }) })