From 5e0efff2f82c2899b3432433a79305ed90356fa6 Mon Sep 17 00:00:00 2001 From: Adam Frisby Date: Fri, 25 Sep 2026 15:58:11 +0000 Subject: [PATCH 1/2] Add lychee auditor plugin for broken link checking New plugin under plugins/auditors-documentation wrapping lychee on the shared ExternalToolAuditorBase. Runs `lychee --format json --offline .` by default: file-scheme links and in-repo fragments are checked deterministically with no credentials and no network; remote URLs are reported as excluded, not findings. CheckRemoteLinks opts in to remote checking and declares AuditCapabilities.Network so the audit sandbox gets egress. lychee's exit convention is not the common one: 0 = clean, 2 = link check failures, 1 = runtime/config errors, 3 = config-file errors; only {0, 2} are findings-producing. error_map entries map to Error, timeout_map entries to Warning, with synthesized rule ids (lychee/broken-link, lychee/timeout) so IncludedRules/ExcludedRules work. Repo-authored suppression fails closed by default: a repo-root .lycheeignore (loaded unconditionally by lychee) is an infrastructure failure, and --config /dev/null pins out the default config-file lookup (lychee.toml and friends). TrustRepositorySuppression or an explicit ConfigPath opts back in. Vendored/generated trees are excluded at crawl time via --exclude-path derived from ExcludePaths and at finding level. The pinned tool version (default 0.24.2) is probed before every run. CodeyBox-Prompt-Revision: 1 Co-Authored-By: CodeyBox --- CodeyBox.slnx | 1 + .../CodeyBox.LycheeAuditorPlugin.csproj | 22 + .../LycheeAuditor.cs | 516 +++++++++++ .../LycheeJsonOutputParser.cs | 177 ++++ .../CodeyBox.LycheeAuditorPlugin/README.md | 162 ++++ tests/CodeyBox.Tests/CodeyBox.Tests.csproj | 1 + tests/CodeyBox.Tests/LycheeAuditorTests.cs | 873 ++++++++++++++++++ 7 files changed, 1752 insertions(+) create mode 100644 plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/CodeyBox.LycheeAuditorPlugin.csproj create mode 100644 plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/LycheeAuditor.cs create mode 100644 plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/LycheeJsonOutputParser.cs create mode 100644 plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/README.md create mode 100644 tests/CodeyBox.Tests/LycheeAuditorTests.cs diff --git a/CodeyBox.slnx b/CodeyBox.slnx index 4391dc9e7..5964af17f 100644 --- a/CodeyBox.slnx +++ b/CodeyBox.slnx @@ -60,6 +60,7 @@ + diff --git a/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/CodeyBox.LycheeAuditorPlugin.csproj b/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/CodeyBox.LycheeAuditorPlugin.csproj new file mode 100644 index 000000000..696fccf93 --- /dev/null +++ b/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/CodeyBox.LycheeAuditorPlugin.csproj @@ -0,0 +1,22 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + + + + diff --git a/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/LycheeAuditor.cs b/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/LycheeAuditor.cs new file mode 100644 index 000000000..6f195666f --- /dev/null +++ b/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/LycheeAuditor.cs @@ -0,0 +1,516 @@ +using System.Text.RegularExpressions; +using CodeyBox.Core; +using CodeyBox.PluginSdk; +using CodeyBox.PluginSdk.Tools; +using Microsoft.Extensions.Logging; + +namespace CodeyBox.LycheeAuditorPlugin; + +/// +/// Documentation auditor wrapping lychee (broken-link checking) on the +/// shared : the base supplies sandboxed +/// invocation with a bounded timeout, per-stream output caps, exit-code +/// classification, severity mapping, finding identity, and per-auditor +/// configuration. This class adds the lychee JSON report parser +/// (), the pinned tool-version probe and +/// the repository-suppression gate via +/// , and the offline/remote +/// posture below. +/// +/// Gate behaviour: hybrid / severity-driven. Confirmed link +/// failures (error_map — dead URLs, missing files, missing fragments) +/// map to and fail the audit. Request +/// timeouts (timeout_map) map to and +/// are advisory only: a timeout is weak evidence of a broken link and should +/// not block on its own. With the default offline scope timeouts cannot +/// occur at all, so the default gate is effectively blocking on every +/// finding. +/// +/// Exit-code convention (verified against lychee v0.24.x). +/// lychee does NOT follow the common "exit 1 = findings" convention: +/// 0 = every non-excluded link checked OK; 2 = link check +/// failures (both emit the JSON report on stdout — both are verdicts); +/// 1 = missing inputs or any other runtime/configuration error; +/// 3 = errors in the config file; 126/127 = cannot +/// execute / not found. Only {0, 2} are declared +/// findings-producing; everything else is infrastructure. A verdict-class +/// exit without a JSON report on stdout still fails closed as infrastructure +/// through the parser. +/// +/// Version pin. A checker's request behaviour, defaults, and +/// report shape change between releases, so findings are only meaningful from +/// the build the auditor was verified against. The auditor probes +/// lychee --version before the scan; a missing binary, an +/// unrecognised version string, or a version other than +/// ExpectedVersion is an infrastructure failure naming the tool — +/// never a pass, never a finding. +/// +/// Offline by default — remote links are not checked. The +/// auditor runs under (no credentials, no +/// network) and passes --offline: lychee restricts checking to the +/// file scheme, so relative file links and in-repo fragments are +/// verified while remote http(s)/mailto links are reported as +/// excluded, not as findings. This keeps the audit deterministic — findings +/// never depend on the availability of an external site. Operators who need +/// remote URL checking set CheckRemoteLinks: the auditor then drops +/// --offline and declares , +/// which provisions network egress for the audit sandbox (subject to the +/// project's audit-tool network profile). Findings in remote mode are +/// inherently non-deterministic. +/// +/// Repository-controlled suppression. lychee honors surfaces +/// authored inside the audited repository: a .lycheeignore file in the +/// working directory is loaded unconditionally (no flag disables it) and its +/// lines become URL-exclusion regexes, and when no --config is given +/// lychee auto-loads configuration from a repo-root lychee.toml (or a +/// [lychee] section in Cargo.toml/pyproject.toml/ +/// package.json). The audit subject writes that repository, so by +/// default a .lycheeignore at the repo root fails closed as +/// infrastructure and the scan pins configuration with +/// --config /dev/null, which disables every default config-file lookup +/// (an operator ConfigPath or --config in ExtraArguments +/// takes precedence over the pin). TrustRepositorySuppression opts in +/// to the repository's own suppression surfaces. Note lychee also skips input +/// files matched by .gitignore/.ignore rules; the auditor keeps +/// that default because --no-ignore would crawl gitignored build +/// output present in the worktree. Residual gap, documented not hidden: a +/// subject could commit a file and then exclude it via .gitignore so +/// lychee's walker skips it — matching a dedicated suppression file's effect +/// through a file nearly every repo legitimately has, which is why it is not +/// gated like .lycheeignore. +/// +/// Scope and defaults. The scan is lychee .: the worktree +/// is walked recursively, filtered to lychee's documentation extensions +/// (Markdown, HTML, CSS, XML, plaintext). Hidden files are included +/// (--hidden) because documentation legitimately lives under +/// dot-directories such as .github/; .git/ is excluded. Links +/// under vendored (vendor/, third_party/, node_modules/) +/// and generated (dist/, build/, out/, coverage/) +/// prefixes are excluded at extraction time (--exclude-path, derived +/// from ExcludePaths) and findings under them are dropped as well: +/// problems there belong to upstream packages or build output, not the +/// change under audit. --include-fragments checks #anchor +/// targets inside local files — a common real-world docs break. Root-absolute +/// links (/docs/x.md) resolve against RootDirectory when +/// configured; otherwise lychee flags them, which is the honest result — the +/// deploy root is unknowable without configuration. +/// +[CodeyBoxPlugin( + id: PluginId, + displayName: "CodeyBox: Lychee Broken Link Checker", + minHostApiVersion: "1.0")] +[CodeyBoxPluginRequiresTool( + "lychee", + InstallHint = "provision the pinned lychee release (see ExpectedVersion, default " + + DefaultExpectedVersion + ") into the sandbox baseline — no distro apt package carries a " + + "version pin — e.g. 'cargo install lychee --locked --version " + DefaultExpectedVersion + + "' or the lychee-v" + DefaultExpectedVersion + + " release tarball via CodeyBox:MultipassExtraRuncmd / CodeyBox:Incus:ExtraRuncmd or ExecutableProvisions")] +public sealed class LycheeAuditor : ExternalToolAuditorBase, IPluginInitializer +{ + /// Plugin id used in Plugins:Enabled and the scoped-config section. + public const string PluginId = "codeybox.lychee"; + + /// + /// lychee release the invocation and its report shape are verified + /// against. Operators running a different pinned build set + /// ExpectedVersion in the plugin's scoped config to match what + /// they provisioned. + /// + public const string DefaultExpectedVersion = "0.24.2"; + + /// Scoped-config key for an explicit lychee configuration file path. + public const string ConfigPathKey = "ConfigPath"; + + /// + /// Scoped-config key for the lychee inputs (comma-separated files, globs, + /// or directories). Replaces the default whole-tree . input. + /// + public const string InputsKey = "Inputs"; + + /// + /// Scoped-config key resolving root-absolute links (/docs/x.md) in + /// local files. Passed to lychee's --root-dir; must be an absolute + /// path inside the sandbox. + /// + public const string RootDirectoryKey = "RootDirectory"; + + /// + /// Scoped-config key opting in to remote link checking. When true the + /// scan drops --offline and the auditor declares + /// so the audit sandbox is + /// provisioned with network egress. + /// + public const string CheckRemoteLinksKey = "CheckRemoteLinks"; + + /// + /// Scoped-config key opting in to repository-authored suppression — + /// .lycheeignore and the default lychee config files + /// (lychee.toml and friends). Default false: the audited repo must + /// not be able to silence the audit. + /// + internal const string TrustRepositorySuppressionKey = "TrustRepositorySuppression"; + + /// Synthesized rule id for error_map entries (lychee reports no rule ids). + internal const string BrokenLinkRuleId = "lychee/broken-link"; + + /// Synthesized rule id for timeout_map entries. + internal const string TimeoutRuleId = "lychee/timeout"; + + // Passing any --config disables lychee's default config-file lookup + // entirely (the default only runs when no --config is given). /dev/null + // parses as an empty config, pinning built-in defaults so repo-authored + // lychee.toml / Cargo.toml / pyproject.toml / package.json sections cannot + // steer the audit. An operator ConfigPath or ExtraArguments --config + // outranks the pin; TrustRepositorySuppression drops it. + private const string PinnedEmptyConfigPath = "/dev/null"; + + // Loaded unconditionally from the process working directory; there is no + // flag to redirect or disable it, so presence at the repo root is gated. + private const string RepositoryIgnoreFile = ".lycheeignore"; + + private const int ProbeMaxOutputBytes = 16 * 1024; + private const int MessageValueMaxChars = 64; + // The precondition probes are liveness checks, not the scan: they never + // need more than this and share the operator-configured timeout below it. + private static readonly TimeSpan ProbeTimeoutCap = TimeSpan.FromSeconds(30); + private static readonly Regex VersionPattern = new( + @"\d+\.\d+\.\d+[\w.\-]*", + RegexOptions.CultureInvariant | RegexOptions.Compiled); + + private static readonly ExternalToolAuditorOptions AuditorDefaults = new() + { + // 0 = all checked links OK; 2 = link check failures. Both emit the + // JSON report — both are verdicts. 1 (missing input / runtime / + // config error), 3 (config-file error), and everything else is + // infrastructure. + FindingsExitCodes = new HashSet { 0, 2 }, + // Findings in VCS internals, vendored/dependency trees, and generated + // build output do not describe the change under audit — noise that + // trains operators to ignore the auditor. Each entry is also passed + // to lychee as --exclude-path so those files are never crawled. + // Operators re-include a path by overriding ExcludePaths in scoped + // config. + ExcludePaths = + [ + ".git/", + "vendor/", + "third_party/", + "node_modules/", + "dist/", + "build/", + "out/", + "coverage/", + ], + }; + + private Func _optionsAccessor = () => AuditorDefaults; + private Func _expectedVersion = static () => DefaultExpectedVersion; + private Func _configPath = static () => null; + private Func _rootDirectory = static () => null; + private Func> _inputs = static () => []; + private Func _checkRemoteLinks = static () => false; + private Func _trustRepositorySuppression = static () => false; + + /// + public override string Name => "codeybox:lychee"; + + /// + protected override string ToolName => "lychee"; + + /// + protected override IExternalToolOutputParser OutputParser { get; } = new LycheeJsonOutputParser(); + + /// + /// Offline by default: the audit sandbox runs this auditor with no + /// credentials and no network. When the operator enables + /// CheckRemoteLinks the auditor declares + /// so the sandbox is provisioned + /// with egress. + /// + public override AuditCapabilities Required => + _checkRemoteLinks() ? AuditCapabilities.Network : AuditCapabilities.None; + + /// + /// Declared mapping from lychee's failure vocabulary to + /// : error_map entries ("error") are + /// confirmed link failures — ; + /// timeout_map entries ("timeout") are weak, flaky evidence — + /// . Anything the parser cannot + /// classify keeps the fail-closed + /// default. Raw tool levels never reach findings. + /// + protected override ExternalToolSeverityMapping SeverityMapping { get; } = + new(new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["error"] = AuditSeverity.Error, + ["timeout"] = AuditSeverity.Warning, + }, AuditSeverity.Error); + + /// + protected override Func OptionsAccessor => _optionsAccessor; + + /// + protected override IReadOnlyList BuildToolArguments(ExternalToolAuditorOptions options) + { + var args = new List + { + // The stats report on stdout is the verdict; the parser reads + // error_map/timeout_map which are populated on every run. + "--format", "json", + "--no-progress", + // Docs legitimately live under dot-directories (.github/, .gitlab/). + // .git/ stays out via the ExcludePaths-derived --exclude-path below. + "--hidden", + // Broken in-repo #anchors are a common docs failure and cost no + // network requests for local targets. + "--include-fragments", + }; + + // The default sandbox has no egress; without this every remote URL + // would be a network-error finding instead of an exclusion. + if (!_checkRemoteLinks()) + args.Add("--offline"); + + // Config precedence: explicit operator config, else the pinned empty + // config that disables repo config loading, else (when trusting the + // repository) lychee's own default lookup. + var configPath = _configPath(); + if (!string.IsNullOrWhiteSpace(configPath) + && !options.ExtraArguments.Contains("--config", StringComparer.Ordinal) + && !options.ExtraArguments.Contains("-c", StringComparer.Ordinal)) + { + args.Add("--config"); + args.Add(configPath.Trim()); + } + else if (!_trustRepositorySuppression() + && !options.ExtraArguments.Contains("--config", StringComparer.Ordinal) + && !options.ExtraArguments.Contains("-c", StringComparer.Ordinal)) + { + args.Add("--config"); + args.Add(PinnedEmptyConfigPath); + } + + var rootDirectory = _rootDirectory(); + if (!string.IsNullOrWhiteSpace(rootDirectory) + && !options.ExtraArguments.Contains("--root-dir", StringComparer.Ordinal)) + { + args.Add("--root-dir"); + args.Add(rootDirectory.Trim()); + } + + // Keep excluded trees out of the crawl entirely — links inside them + // would be checked (wasted requests) before the findings-level + // ExcludePaths filter could drop them. Each entry is translated into + // an anchored regex matching the base's exact/prefix semantics. + foreach (var entry in options.ExcludePaths) + { + var pattern = ToExcludePathPattern(entry); + if (pattern is not null) + { + args.Add("--exclude-path"); + args.Add(pattern); + } + } + + var inputs = _inputs(); + if (inputs.Count > 0) + args.AddRange(inputs); + else + args.Add("."); + return args; + } + + /// + public Task InitializeAsync(PluginContext context, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(context); + var scoped = context.ScopedConfig; + _optionsAccessor = () => ExternalToolAuditorOptions.Bind(scoped, AuditorDefaults); + _expectedVersion = () => scoped["ExpectedVersion"]; + _configPath = () => scoped[ConfigPathKey]; + _rootDirectory = () => scoped[RootDirectoryKey]; + _inputs = () => SplitList(scoped[InputsKey]); + _checkRemoteLinks = () => + bool.TryParse(scoped[CheckRemoteLinksKey], out var check) && check; + _trustRepositorySuppression = () => + bool.TryParse(scoped[TrustRepositorySuppressionKey], out var trust) && trust; + context.Logger.LogInformation( + "LycheeAuditor initialized: pluginId={PluginId}", context.PluginId); + return Task.CompletedTask; + } + + /// + /// lychee-specific preconditions on the live path: the pinned scanner + /// version, and — unless the operator opted in — absence of a repo-root + /// .lycheeignore, which lychee would load unconditionally. Both + /// fail closed as infrastructure before the scan runs. + /// + protected override async Task VerifyToolAsync( + ISandbox sandbox, + string workingDirectory, + string tool, + ExternalToolAuditorOptions options, + CancellationToken ct) + { + await ThrowIfToolVersionMismatchAsync(sandbox, workingDirectory, tool, options, ct) + .ConfigureAwait(false); + if (!_trustRepositorySuppression()) + await ThrowIfRepositoryIgnoreFilePresentAsync(sandbox, workingDirectory, tool, options, ct) + .ConfigureAwait(false); + } + + private async Task ThrowIfToolVersionMismatchAsync( + ISandbox sandbox, + string workingDirectory, + string tool, + ExternalToolAuditorOptions options, + CancellationToken ct) + { + var configured = _expectedVersion(); + var expected = NormalizeVersion(configured); + if (expected is null) + throw new AuditUnavailableException( + $"could-not-verify: auditor '{Name}' has an unparseable ExpectedVersion " + + $"('{TruncateForMessage(configured)}'); set CodeyBox:Plugins:{PluginId}:ExpectedVersion " + + $"to a {tool} release such as '{DefaultExpectedVersion}'.") + { IsDeterministic = true }; + + var result = await ExecToolBoundedAsync( + sandbox, + tool, + "version check", + new SandboxExec + { + Argv = [tool, "--version"], + WorkingDirectory = workingDirectory, + MaxStdoutBytes = ProbeMaxOutputBytes, + MaxStderrBytes = ProbeMaxOutputBytes, + KillOnOutputLimit = true, + }, + ProbeTimeout(options), + ct).ConfigureAwait(false); + + var reported = ExtractVersion(result.Stdout); + if (result.ExecutionUnavailable + || result.ExitCode != 0 + || reported is null) + throw new AuditUnavailableException( + $"could-not-verify: audit tool '{tool}' version could not be determined " + + $"(exit {result.ExitCode}). The pinned release is required before the scan can run — " + + $"a missing or foreign '{tool}' is infrastructure, not a verdict on the diff.", + result.ExitCode, + result.Stdout + "\n" + result.Stderr); + + if (!string.Equals(reported, expected, StringComparison.Ordinal)) + throw new AuditUnavailableException( + $"could-not-verify: audit tool '{tool}' is version {reported}, but this auditor is " + + $"pinned to {expected}. A different checker version changes the link checks and the " + + "findings; provision the pinned release or set ExpectedVersion to the version you provisioned.") + { IsDeterministic = true }; + } + + private async Task ThrowIfRepositoryIgnoreFilePresentAsync( + ISandbox sandbox, + string workingDirectory, + string tool, + ExternalToolAuditorOptions options, + CancellationToken ct) + { + // lychee loads /.lycheeignore unconditionally — its lines become + // URL-exclusion regexes the audit subject could use to hide a broken + // link — and no flag redirects or disables that load, so presence is + // gated rather than outranked. (-L covers a dangling symlink that -e + // would miss.) + var presence = await ExecToolBoundedAsync( + sandbox, + tool, + "suppression check", + new SandboxExec + { + Argv = ["sh", "-c", "[ -e \"$1\" ] || [ -L \"$1\" ]", "sh", RepositoryIgnoreFile], + WorkingDirectory = workingDirectory, + MaxStdoutBytes = ProbeMaxOutputBytes, + MaxStderrBytes = ProbeMaxOutputBytes, + KillOnOutputLimit = true, + }, + ProbeTimeout(options), + ct).ConfigureAwait(false); + + if (presence.ExecutionUnavailable) + throw new AuditUnavailableException( + $"could-not-verify: audit tool '{tool}' suppression check could not run: the sandbox exec " + + "transport was unavailable."); + if (presence.ExitCode == 0) + throw new AuditUnavailableException( + $"could-not-verify: audit tool '{tool}' found '{RepositoryIgnoreFile}' at the root of the " + + "audited repository — lychee loads it unconditionally and honors its URL-exclusion " + + "patterns, so the audit subject could use it to hide a broken link. Remove the file, " + + "or set CodeyBox:Plugins:" + PluginId + ":" + TrustRepositorySuppressionKey + + " to true to trust repository-controlled suppression surfaces.", + presence.ExitCode, + presence.Stdout + "\n" + presence.Stderr) + { IsDeterministic = true }; + } + + /// + /// Translates an + /// entry into a lychee --exclude-path regex preserving the base's + /// semantics: a trailing-/ entry excludes everything beneath that + /// root-level directory prefix; any other entry excludes exactly that + /// root-relative path. Anchored at the start (allowing the walker-emitted + /// "./" prefix) so nested namesakes like docs/vendor/ are not + /// swept up. Every entry is d — operator config + /// is treated as data, never as pattern text. + /// + internal static string? ToExcludePathPattern(string? entry) + { + if (string.IsNullOrWhiteSpace(entry)) + return null; + var normalized = entry.Replace('\\', '/').Trim().TrimStart('/'); + if (normalized.Length == 0) + return null; + return normalized.EndsWith('/') + ? @"^(?:\./)?" + Regex.Escape(normalized) + : @"^(?:\./)?" + Regex.Escape(normalized) + "$"; + } + + private static TimeSpan ProbeTimeout(ExternalToolAuditorOptions options) + { + var timeout = EffectiveTimeout(options); + return timeout > ProbeTimeoutCap ? ProbeTimeoutCap : timeout; + } + + private static string? ExtractVersion(string stdout) + { + // `lychee --version` prints "lychee 0.24.2". + var match = VersionPattern.Match(stdout); + return match.Success ? match.Value : null; + } + + private static string? NormalizeVersion(string? configured) + { + var value = string.IsNullOrWhiteSpace(configured) + ? DefaultExpectedVersion + : configured.Trim(); + var match = VersionPattern.Match(value); + return match.Success ? match.Value : null; + } + + private static string TruncateForMessage(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + return "(empty)"; + var single = SingleLine(value); + return single.Length > MessageValueMaxChars + ? single[..MessageValueMaxChars] + "…" + : single; + } + + private static List SplitList(string? value) + => string.IsNullOrWhiteSpace(value) + ? [] + : value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Where(static item => item.Length > 0) + .ToList(); +} diff --git a/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/LycheeJsonOutputParser.cs b/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/LycheeJsonOutputParser.cs new file mode 100644 index 000000000..eaa721ec0 --- /dev/null +++ b/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/LycheeJsonOutputParser.cs @@ -0,0 +1,177 @@ +using System.Text.Json; +using CodeyBox.PluginSdk.Tools; + +namespace CodeyBox.LycheeAuditorPlugin; + +/// +/// Parses lychee's --format json status report into +/// records. The report is a stats object +/// whose failure detail lives in two maps keyed by input source (the file the +/// link was extracted from, e.g. "./docs/guide.md", or "stdin"/a +/// URL for non-file inputs): +/// +/// +/// "error_map": { "<source>": [ { "url": …, "status": { "text": …, "code": 404, "details": … }, "span": { "line": …, "column": … } } ] }, +/// "timeout_map": { "<source>": [ … same entry shape … ] } +/// +/// +/// Both maps are populated on every run (no verbosity flag needed) and both +/// drive lychee's exit code: error_map entries are confirmed link +/// failures, timeout_map entries are requests that timed out. Each +/// keeps its own vocabulary level ("error" / "timeout") so the +/// auditor's declared decides their +/// meaning; raw levels never reach findings. success_map, +/// redirect_map, excluded_map, and suggestion_map are not +/// failures — they never become findings. +/// +/// Rule ids are synthesized because lychee reports none: +/// for error_map entries +/// and for timeout_map +/// entries, so the shared IncludedRules/ExcludedRules +/// configuration works on a stable identity. +/// +/// span.line is carried through as the finding line when lychee +/// supplies it. Source keys are normalized by stripping the walker-emitted +/// "./" prefix so the shared ExcludePaths prefix matching lines +/// up with repository-relative paths. +/// +/// Malformed output — empty stdout, non-JSON, or a JSON object without +/// lychee's total/error_map shape (e.g. a usage dump or a +/// report in a different format after an ExtraArguments +/// --format override) — throws , +/// which the base reports as infrastructure, never as a pass. +/// +internal sealed class LycheeJsonOutputParser : IExternalToolOutputParser +{ + // Same per-document result bound the shared SARIF parser applies. + private const int MaxResults = SarifToolOutputParser.DefaultMaxResults; + private const int MessageMaxChars = 512; + + public IReadOnlyList Parse(ExternalToolParseInput input) + { + ArgumentNullException.ThrowIfNull(input); + if (string.IsNullOrWhiteSpace(input.Stdout)) + throw new ExternalToolParseException( + $"Tool '{input.ToolName}' produced no lychee JSON output on stdout."); + + JsonDocument document; + try + { + document = JsonDocument.Parse(input.Stdout); + } + catch (JsonException ex) + { + throw new ExternalToolParseException( + $"Tool '{input.ToolName}' produced output that is not valid JSON: {SingleLine(ex.Message)}.", + ex); + } + + using (document) + { + var root = document.RootElement; + if (root.ValueKind != JsonValueKind.Object + || !root.TryGetProperty("total"u8, out var total) + || total.ValueKind != JsonValueKind.Number + || !root.TryGetProperty("error_map"u8, out var errorMap) + || errorMap.ValueKind != JsonValueKind.Object) + throw new ExternalToolParseException( + $"Tool '{input.ToolName}' produced JSON that is not a lychee report " + + "(missing the 'total'/'error_map' stats shape)."); + + var findings = new List(); + CollectMap(errorMap, LycheeAuditor.BrokenLinkRuleId, "error", findings); + if (root.TryGetProperty("timeout_map"u8, out var timeoutMap) + && timeoutMap.ValueKind == JsonValueKind.Object) + CollectMap(timeoutMap, LycheeAuditor.TimeoutRuleId, "timeout", findings); + return findings; + } + } + + private static void CollectMap( + JsonElement map, + string ruleId, + string level, + List findings) + { + foreach (var property in map.EnumerateObject()) + { + if (findings.Count >= MaxResults) + return; + if (property.Value.ValueKind != JsonValueKind.Array) + continue; + + var path = NormalizeSourcePath(property.Name); + foreach (var entry in property.Value.EnumerateArray()) + { + if (findings.Count >= MaxResults) + return; + if (entry.ValueKind == JsonValueKind.Object) + findings.Add(ParseEntry(entry, ruleId, level, path)); + } + } + } + + private static ExternalToolFinding ParseEntry( + JsonElement entry, + string ruleId, + string level, + string? path) + { + var url = GetString(entry, "url"u8); + var statusText = GetStatusText(entry); + + int? line = null; + if (entry.TryGetProperty("span"u8, out var span) + && span.ValueKind == JsonValueKind.Object + && span.TryGetProperty("line"u8, out var lineElement) + && lineElement.ValueKind == JsonValueKind.Number + && lineElement.TryGetInt32(out var lineValue) + && lineValue > 0) + line = lineValue; + + var message = (url ?? "(unknown url)") + " — " + (statusText ?? "link check failed"); + return new ExternalToolFinding( + SeverityLevel: level, + RuleId: ruleId, + Message: message.Length > MessageMaxChars ? message[..MessageMaxChars] + "…" : message, + Path: path, + Line: line); + } + + private static string? GetStatusText(JsonElement entry) + { + if (!entry.TryGetProperty("status"u8, out var status) + || status.ValueKind != JsonValueKind.Object) + return null; + + var text = GetString(status, "text"u8); + var details = GetString(status, "details"u8); + if (text is null) + return details; + return details is null ? text : text + "; " + details; + } + + private static string? GetString(JsonElement element, ReadOnlySpan name) + => element.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String + ? NullIfWhiteSpace(value.GetString()) + : null; + + // Input sources arrive as the walker reported them ("./docs/guide.md", + // "README.md", "stdin", a URL for remote inputs). Strip the "./" prefix so + // ExcludePaths prefix matching sees repository-relative paths. + private static string? NormalizeSourcePath(string? raw) + { + if (string.IsNullOrWhiteSpace(raw)) + return null; + var path = raw.Trim().Replace('\\', '/'); + while (path.StartsWith("./", StringComparison.Ordinal)) + path = path[2..]; + return path.Length == 0 ? null : path; + } + + private static string? NullIfWhiteSpace(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value; + + private static string SingleLine(string message) + => message.Replace('\r', ' ').Replace('\n', ' ').Trim(); +} diff --git a/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/README.md b/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/README.md new file mode 100644 index 000000000..ec9c063bb --- /dev/null +++ b/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/README.md @@ -0,0 +1,162 @@ +# CodeyBox: Lychee Broken-Link Auditor + +Auditor plugin wrapping [lychee](https://lychee.cli.rs): it checks links in +the audited repository's documentation files with `lychee --format json +--offline .` and reports each failed link check as an audit finding with the +source file and `file:line` location lychee supplies. + +## What it reports + +- One finding per failed link check. lychee's JSON report carries no rule + ids, so the auditor synthesizes two stable ones: + - `lychee/broken-link` — an `error_map` entry: a link that was checked and + failed (missing file, dead URL, missing `#fragment`). Maps to + `AuditSeverity.Error`. + - `lychee/timeout` — a `timeout_map` entry: a request that timed out. Maps + to `AuditSeverity.Warning` — a timeout is weak evidence of a broken link + and stays advisory. +- The finding title carries the synthesized rule id and `url — status`; the + description carries the tool, rule, tool-reported level, location, and the + full message. `Location` is `path:span.line` when lychee reports a span. +- **Gate behaviour: hybrid / severity-driven.** `lychee/broken-link` findings + fail the audit; `lychee/timeout` findings are advisory. With the default + offline scope timeouts cannot occur, so the default gate is effectively + blocking on every finding. +- `success_map`, `redirect_map`, `excluded_map`, and `suggestion_map` are + not failures and never become findings. + +## What it cannot see + +- **Remote links by default.** The auditor runs under + `AuditCapabilities.None` — no credentials, no network — and passes + `--offline`, so lychee restricts checking to the `file` scheme: relative + file links and in-repo `#fragment` targets are verified; `http(s)`, + `mailto`, and other remote URLs are excluded, not reported. Set + `CheckRemoteLinks` (below) to opt in — the auditor then declares + `AuditCapabilities.Network` and the sandbox is provisioned with egress + subject to the project's audit-tool network profile. Remote findings are + inherently non-deterministic (a flaky site produces flaky findings). +- **Line numbers lychee does not emit.** `span.line` is reported per link in + v0.24.x JSON; entries without a span produce findings with no line. +- **Files outside the walked extensions.** The `.` input is filtered to + lychee's documentation extensions (`md`/`mkd`/`mdx`/`mdown`/`mdwn`/`mkdn`/ + `mkdown`/`markdown`, `html`/`htm`, `css`, `txt`, `xml`). Links inside + `*.py`, `*.cs`, `*.yml`, etc. are not extracted. +- **Gitignored and excluded inputs.** lychee skips files covered by + `.gitignore`/`.ignore`, and paths listed in `ExcludePaths` are excluded at + crawl time via `--exclude-path` — links inside them are never checked. +- **Site-root-absolute links resolve only with `RootDirectory`.** A link like + `/docs/guide.md` is only checkable if the deploy root is known; without + `RootDirectory` lychee reports it — which is the honest result. + +## Exit codes and failure classification + +lychee's convention (verified against v0.24.x — **not** the common +"1 = findings" pattern): + +| Exit | Meaning | Classification | +|---|---|---| +| `0` | All non-excluded links checked OK | Verdict (pass) | +| `2` | Link check failures (`error_map`/`timeout_map` non-empty) | Verdict (`Passed = false` when any finding maps to `Error`) | +| `1` | Missing inputs, runtime failure, or general config error | Infrastructure (`AuditUnavailableException`) | +| `3` | Errors in the lychee config file | Infrastructure | +| `0`/`2` with no JSON on stdout | Execution failure, or a `--format` override that changed the report shape | Infrastructure — the JSON parser fails closed | +| `126` / `127` | Binary not executable or not found | Infrastructure | +| anything else | Unknown convention | Infrastructure (fails loud, never a pass) | + +A missing `lychee` is always an infrastructure failure naming the tool — +never a passing audit. + +## Version pinning + +The auditor is pinned to **lychee `0.24.2`** (`ExpectedVersion` in scoped +config). A checker's request behaviour, defaults, and report shape change +between releases, so an unpinned tool would change findings under you: the +auditor probes `lychee --version` before every run and reports an +infrastructure failure on any other version. + +The tool requirement is declared **verify-only** — no `AptPackage`: lychee is +not packaged by Ubuntu/Debian and the requirement must carry a version pin. +Provision the pinned release in your sandbox baseline, e.g.: + +```sh +# baseline bake step +cargo install lychee --locked --version 0.24.2 +# or unpack the lychee-v0.24.2 release tarball from the project's releases +lychee --version # must print "lychee 0.24.2" +``` + +## Enabling + +The plugin is **disabled by default** — it loads only when named in both +gates: + +```json +{ + "CodeyBox": { + "Plugins": { + "Allowlist": ["codeybox.lychee"], + "Enabled": ["codeybox.lychee"] + } + } +} +``` + +and per project under `Audit.Custom`: + +```json +{ "Kind": "plugin", "PluginId": "codeybox.lychee" } +``` + +The `lychee` tool requirement is only contributed to baseline provisioning +while the plugin is enabled. + +## Configuration + +Scoped under `CodeyBox:Plugins:codeybox.lychee`, resolved per run +(hot-reloadable): + +| Key | Default | Meaning | +|---|---|---| +| `ExpectedVersion` | `0.24.2` | Pinned lychee release; a different installed version fails closed as infrastructure. Set this to the release you provisioned. | +| `CheckRemoteLinks` | `false` | `true` drops `--offline` and declares `AuditCapabilities.Network` so the audit sandbox gets egress. Remote findings are non-deterministic — timeouts and site outages become part of the verdict. | +| `RootDirectory` | `null` | Absolute in-sandbox path passed to `--root-dir`; resolves site-root-absolute links (`/docs/x.md`) in local files. Ignored when `ExtraArguments` supplies `--root-dir`. | +| `Inputs` | `.` | Comma-separated lychee inputs (files, globs, directories) replacing the whole-tree `.` default — e.g. `docs,README.md` to scope to one tree. | +| `ConfigPath` | `null` | Path passed to `--config` — an operator-pinned lychee config. Takes precedence over the `/dev/null` pin and the repo's own config files. Ignored when `ExtraArguments` already supplies `--config`/`-c`. | +| `TrustRepositorySuppression` | `false` | When `false` (default) a repo-root `.lycheeignore` fails closed and `--config /dev/null` disables all default config-file lookup. When `true`, lychee honors `.lycheeignore` and repo-authored `lychee.toml`/`[lychee]` sections. | +| `MinimumSeverity` | `info` | Drop mapped findings below this severity (`info`, `warning`, `error`). | +| `IncludedRules` / `ExcludedRules` | — | Exact rule ids to keep/drop; the only ids are `lychee/broken-link` and `lychee/timeout`. | +| `ExcludePaths` | `.git/`, `vendor/`, `third_party/`, `node_modules/`, `dist/`, `build/`, `out/`, `coverage/` | Repo-relative paths dropped from findings — exact path, or directory prefix when trailing `/`. Each entry is also translated to an anchored `--exclude-path` regex so excluded trees are never crawled. Setting it replaces the default list. | +| `ExtraArguments` | — | Extra argv appended after the built-in args (never via a shell). Useful for `--exclude `, `--accept `, `--scheme https`, or an operator `--config`/`--exclude-path`. A repeated flag wins over the built-in default — take care: `--format` would replace the JSON report the parser expects and break the run into infrastructure failure. | +| `TimeoutSeconds` | `300` | Per-run bound. Exceeding it is infrastructure, not a pass. | +| `MaxOutputBytesPerStream` / `MaxFindings` | `1 MiB` / `1000` | Output/result caps; overruns are reported as truncation. | + +**Repository-controlled suppression is off by default.** The audit subject +writes the repository, and lychee honors two repo-authored surfaces: +`.lycheeignore` in the working directory (loaded unconditionally — no flag +disables it — each line a URL-exclusion regex) and the default config files +(`lychee.toml`, or `[lychee]`-equivalent sections in `Cargo.toml`, +`pyproject.toml`, `package.json`), which can widen `exclude`/`accept` and +silence findings. By default the auditor fails closed when a repo-root +`.lycheeignore` exists, and passes `--config /dev/null` — an empty config +that turns off every default lookup — so the checked ruleset cannot be +steered by the audited tree. `TrustRepositorySuppression: true` restores the +tool's default behavior for repos whose own lychee config is the intended +contract; a deliberate `ConfigPath` overrides the pin without trusting the +repo. + +If your repository legitimately ships `lychee.toml` or `.lycheeignore` for its +own CI, either point `ConfigPath` at that file (it is then the explicit, +operator-chosen config) or set `TrustRepositorySuppression: true`. + +## Default scope + +`lychee .` — the whole worktree, walked recursively and filtered to +documentation extensions, with hidden files included (docs live under +`.github/`) and `.git/` excluded. Findings under vendored +(`vendor/`, `third_party/`, `node_modules/`) and generated (`dist/`, +`build/`, `out/`, `coverage/`) prefixes are excluded both at crawl time +(`--exclude-path`) and at finding level (`ExcludePaths`): broken links in +third-party docs or build output belong to upstream packages, not the change +under audit — reporting them trains operators to ignore the auditor. +`Inputs` narrows the crawl itself when whole-tree coverage is too broad. diff --git a/tests/CodeyBox.Tests/CodeyBox.Tests.csproj b/tests/CodeyBox.Tests/CodeyBox.Tests.csproj index 63316a409..49f6a133b 100644 --- a/tests/CodeyBox.Tests/CodeyBox.Tests.csproj +++ b/tests/CodeyBox.Tests/CodeyBox.Tests.csproj @@ -49,6 +49,7 @@ + diff --git a/tests/CodeyBox.Tests/LycheeAuditorTests.cs b/tests/CodeyBox.Tests/LycheeAuditorTests.cs new file mode 100644 index 000000000..b43edb5d2 --- /dev/null +++ b/tests/CodeyBox.Tests/LycheeAuditorTests.cs @@ -0,0 +1,873 @@ +using System.Diagnostics; +using System.Text.RegularExpressions; +using CodeyBox.Core; +using CodeyBox.LycheeAuditorPlugin; +using CodeyBox.Orchestrator; +using CodeyBox.PluginSdk; +using CodeyBox.Sandbox.Process; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; + +namespace CodeyBox.Tests; + +/// +/// Covers the lychee auditor plugin: +/// - Missing or wrong-version binary is an infrastructure failure naming lychee (never a pass or finding). +/// - Exit codes 0 and 2 are verdicts (findings-producing); exit 1/3 and others are infrastructure. +/// - A verdict-class exit without lychee JSON output fails closed as an infrastructure failure. +/// - lychee JSON output maps error_map/timeout_map to findings with rule ids, locations, and +/// mapped severity; "./"-prefixed source paths are normalized to repo-relative. +/// - Raw tool levels ("error"/"timeout") go through the declared mapping. +/// - Default scope ("." input, --hidden, ExcludePaths → --exclude-path), the /dev/null config pin, +/// the .lycheeignore presence gate, and scoped options (ExpectedVersion, ConfigPath, Inputs, +/// RootDirectory, CheckRemoteLinks, TrustRepositorySuppression). +/// - Offline by default (AuditCapabilities.None); CheckRemoteLinks drops --offline and +/// declares Network. +/// - Plugin is disabled by default, absent from baseline provisioning until enabled. +/// - Real binary execution tests under [Trait("requires_lychee", "true")]. +/// +public sealed class LycheeAuditorTests +{ + private static readonly string? InstalledLycheeVersion = ProbeInstalledLycheeVersion(); + + private const string JsonWithBrokenLinks = """ + { + "total": 5, + "unique": 5, + "successful": 2, + "unknown": 0, + "unsupported": 0, + "timeouts": 1, + "redirects": 0, + "remaps": 0, + "excludes": 0, + "errors": 2, + "cached": 0, + "success_map": {}, + "error_map": { + "./docs/guide.md": [ + { + "url": "https://example.com/missing", + "status": { "text": "404 Not Found", "code": 404 }, + "span": { "line": 12, "column": 5 }, + "duration": { "secs": 0, "nanos": 1 } + } + ], + "./README.md": [ + { + "url": "./missing-file.md", + "status": { "text": "Cannot find file", "details": "cannot find file" }, + "span": { "line": 3, "column": 10 } + } + ] + }, + "timeout_map": { + "./docs/guide.md": [ + { + "url": "https://slow.example.com/", + "status": { "text": "Timeout", "details": "Request timed out" }, + "span": { "line": 20, "column": 1 } + } + ] + }, + "suggestion_map": {}, + "redirect_map": {}, + "excluded_map": {}, + "duration": { "secs": 1, "nanos": 0 }, + "detailed_stats": false + } + """; + + private const string JsonClean = """ + { + "total": 3, + "unique": 3, + "successful": 3, + "unknown": 0, + "unsupported": 0, + "timeouts": 0, + "redirects": 0, + "remaps": 0, + "excludes": 0, + "errors": 0, + "cached": 0, + "success_map": {}, + "error_map": {}, + "timeout_map": {}, + "suggestion_map": {}, + "redirect_map": {}, + "excluded_map": {}, + "duration": { "secs": 0, "nanos": 1 }, + "detailed_stats": false + } + """; + + private const string JsonWithFilteredPaths = """ + { + "total": 4, + "unique": 4, + "successful": 1, + "unknown": 0, + "unsupported": 0, + "timeouts": 0, + "redirects": 0, + "remaps": 0, + "excludes": 0, + "errors": 3, + "cached": 0, + "success_map": {}, + "error_map": { + "./vendor/lib/README.md": [ + { "url": "https://example.com/a", "status": { "text": "404 Not Found", "code": 404 }, "span": { "line": 1, "column": 1 } } + ], + "./node_modules/pkg/README.md": [ + { "url": "https://example.com/b", "status": { "text": "404 Not Found", "code": 404 }, "span": { "line": 2, "column": 1 } } + ], + "./docs/guide.md": [ + { "url": "./gone.md", "status": { "text": "Cannot find file" }, "span": { "line": 7, "column": 2 } } + ] + }, + "timeout_map": {}, + "suggestion_map": {}, + "redirect_map": {}, + "excluded_map": {}, + "duration": { "secs": 0, "nanos": 1 }, + "detailed_stats": false + } + """; + + [Fact] + public async Task MissingBinary_IsInfrastructureFailure_NamingLychee_NeverAPass() + { + var scanExecs = 0; + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec)) + return Task.FromResult(new SandboxExecResult(1, "", "")); + if (IsVersionProbe(exec)) + return Task.FromResult(new SandboxExecResult(127, "", "lychee: command not found")); + scanExecs++; + return Task.FromResult(new SandboxExecResult(0, JsonClean, "")); + }); + + IAuditor auditor = new LycheeAuditor(); + var ex = await Assert.ThrowsAsync( + () => auditor.RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None)); + + Assert.Contains("lychee", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, scanExecs); + } + + [Fact] + public async Task VersionProbeFailed_IsInfrastructureFailure_NamingLychee() + { + var scanExecs = 0; + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec)) + return Task.FromResult(new SandboxExecResult(0, "", "")); + if (IsVersionProbe(exec)) + return Task.FromResult(new SandboxExecResult(127, "", "")); + scanExecs++; + return Task.FromResult(new SandboxExecResult(0, JsonClean, "")); + }); + + IAuditor auditor = new LycheeAuditor(); + var ex = await Assert.ThrowsAsync( + () => auditor.RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None)); + + Assert.Contains("lychee", ex.Message, StringComparison.Ordinal); + Assert.Contains("could not be determined", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, scanExecs); + } + + [Fact] + public async Task WrongVersion_IsInfrastructureFailure() + { + var scanExecs = 0; + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec)) + return Task.FromResult(new SandboxExecResult(0, "", "")); + if (IsVersionProbe(exec)) + return Task.FromResult(new SandboxExecResult(0, "lychee 0.99.0\n", "")); + scanExecs++; + return Task.FromResult(new SandboxExecResult(0, JsonClean, "")); + }); + + IAuditor auditor = new LycheeAuditor(); + var ex = await Assert.ThrowsAsync( + () => auditor.RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None)); + + Assert.Contains("lychee", ex.Message, StringComparison.Ordinal); + Assert.Contains("0.99.0", ex.Message, StringComparison.Ordinal); + Assert.Contains(LycheeAuditor.DefaultExpectedVersion, ex.Message, StringComparison.Ordinal); + Assert.Equal(0, scanExecs); + } + + [Fact] + public async Task UnparseableExpectedVersion_IsInfrastructureFailure() + { + var scanExecs = 0; + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec) || IsIgnoreProbe(exec)) + return Task.FromResult(Ok(exec)); + scanExecs++; + return Task.FromResult(new SandboxExecResult(0, JsonClean, "")); + }); + + var auditor = new LycheeAuditor(); + await auditor.InitializeAsync( + BuildPluginContext(new Dictionary + { + ["Scoped:ExpectedVersion"] = "not-a-valid-version-string", + }), + CancellationToken.None); + + var ex = await Assert.ThrowsAsync( + () => ((IAuditor)auditor).RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None)); + + Assert.Contains("unparseable ExpectedVersion", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, scanExecs); + } + + [Fact] + public async Task Fixture_WithBrokenLinks_YieldsFindings_WithRuleIdAndLocation() + { + SandboxExec? scanExec = null; + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec) || IsIgnoreProbe(exec)) + return Task.FromResult(Ok(exec)); + scanExec = exec; + return Task.FromResult(new SandboxExecResult(2, JsonWithBrokenLinks, "")); + }); + + IAuditor auditor = new LycheeAuditor(); + var result = await auditor.RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None); + + Assert.False(result.Passed); + Assert.Equal(3, result.Findings.Count); + + var remote404 = Assert.Single(result.Findings, f => f.Title.Contains("example.com/missing", StringComparison.Ordinal)); + Assert.Equal("codeybox:lychee", remote404.AuditorName); + Assert.Equal(AuditSeverity.Error, remote404.Severity); + Assert.Contains(LycheeAuditor.BrokenLinkRuleId, remote404.Title, StringComparison.Ordinal); + // lychee's walker emits "./"-prefixed sources; the parser relativizes. + Assert.Equal("docs/guide.md:12", remote404.Location); + + var missingFile = Assert.Single(result.Findings, f => f.Title.Contains("missing-file.md", StringComparison.Ordinal)); + Assert.Equal(AuditSeverity.Error, missingFile.Severity); + Assert.Equal("README.md:3", missingFile.Location); + + var timeout = Assert.Single(result.Findings, f => f.Title.Contains(LycheeAuditor.TimeoutRuleId, StringComparison.Ordinal)); + Assert.Equal(AuditSeverity.Warning, timeout.Severity); + Assert.Equal("docs/guide.md:20", timeout.Location); + + Assert.NotNull(scanExec); + Assert.Equal("lychee", scanExec!.Argv[0]); + Assert.Contains("--format", scanExec.Argv); + Assert.Contains("json", scanExec.Argv); + Assert.Contains("--no-progress", scanExec.Argv); + Assert.Contains("--hidden", scanExec.Argv); + Assert.Contains("--include-fragments", scanExec.Argv); + // Offline by default — no network in the audit sandbox. + Assert.Contains("--offline", scanExec.Argv); + // Repo-authored lychee config is pinned out; /dev/null parses empty. + Assert.Contains("--config", scanExec.Argv); + Assert.Contains("/dev/null", scanExec.Argv); + // Vendored/generated trees are excluded at crawl time too. + Assert.Contains(@"^(?:\./)?vendor/", scanExec.Argv); + Assert.Contains(@"^(?:\./)?\.git/", scanExec.Argv); + Assert.Equal(".", scanExec.Argv[^1]); + } + + [Fact] + public async Task CleanFixture_YieldsZeroFindings_AndPasses() + { + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec) || IsIgnoreProbe(exec)) + return Task.FromResult(Ok(exec)); + return Task.FromResult(new SandboxExecResult(0, JsonClean, "")); + }); + + IAuditor auditor = new LycheeAuditor(); + var result = await auditor.RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None); + + Assert.True(result.Passed); + Assert.Empty(result.Findings); + } + + [Fact] + public async Task ExitCode2_IsTheFoundSomethingExit_ReportsFindings_NotInfrastructure() + { + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec) || IsIgnoreProbe(exec)) + return Task.FromResult(Ok(exec)); + return Task.FromResult(new SandboxExecResult(2, JsonWithBrokenLinks, "")); + }); + + IAuditor auditor = new LycheeAuditor(); + var result = await auditor.RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None); + + Assert.False(result.Passed); + Assert.NotEmpty(result.Findings); + } + + [Fact] + public async Task ExitCode1_IsInfrastructureFailure_ThrowsAuditUnavailableException() + { + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec) || IsIgnoreProbe(exec)) + return Task.FromResult(Ok(exec)); + // lychee exits 1 on missing inputs and runtime/configuration errors. + return Task.FromResult(new SandboxExecResult(1, "", "Error: no input files found")); + }); + + IAuditor auditor = new LycheeAuditor(); + var ex = await Assert.ThrowsAsync( + () => auditor.RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None)); + + Assert.Contains("lychee", ex.Message, StringComparison.Ordinal); + Assert.Contains("exit 1", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task ExitCode3_IsInfrastructureFailure_ThrowsAuditUnavailableException() + { + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec) || IsIgnoreProbe(exec)) + return Task.FromResult(Ok(exec)); + // lychee exits 3 on config-file errors. + return Task.FromResult(new SandboxExecResult(3, "", "Error while loading config")); + }); + + IAuditor auditor = new LycheeAuditor(); + var ex = await Assert.ThrowsAsync( + () => auditor.RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None)); + + Assert.Contains("lychee", ex.Message, StringComparison.Ordinal); + Assert.Contains("exit 3", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task VerdictExit_WithoutJson_IsInfrastructureFailure() + { + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec) || IsIgnoreProbe(exec)) + return Task.FromResult(Ok(exec)); + // Exit 2 with no JSON report fails closed as infrastructure. + return Task.FromResult(new SandboxExecResult(2, "", "internal error")); + }); + + IAuditor auditor = new LycheeAuditor(); + var ex = await Assert.ThrowsAsync( + () => auditor.RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None)); + + Assert.Contains("lychee", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task ExitCode127_IsInfrastructureFailure() + { + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec) || IsIgnoreProbe(exec)) + return Task.FromResult(Ok(exec)); + return Task.FromResult(new SandboxExecResult(127, "", "lychee: command not found")); + }); + + IAuditor auditor = new LycheeAuditor(); + var ex = await Assert.ThrowsAsync( + () => auditor.RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None)); + + Assert.Contains("lychee", ex.Message, StringComparison.Ordinal); + Assert.Contains("127", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task SeverityMapping_MapsLevelsCorrectly_NoRawStringsPassedThrough() + { + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec) || IsIgnoreProbe(exec)) + return Task.FromResult(Ok(exec)); + return Task.FromResult(new SandboxExecResult(2, JsonWithBrokenLinks, "")); + }); + + IAuditor auditor = new LycheeAuditor(); + var result = await auditor.RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None); + + // "error" (error_map) -> Error; "timeout" (timeout_map) -> Warning. + // Raw lychee vocabulary never reaches the finding severity — it stays + // visible only in the description's "Severity (tool):" line. + Assert.Equal(2, result.Findings.Count(f => f.Severity == AuditSeverity.Error)); + var timeout = Assert.Single(result.Findings, f => f.Severity == AuditSeverity.Warning); + Assert.Contains("Severity (tool): timeout", timeout.Description, StringComparison.Ordinal); + } + + [Fact] + public async Task LycheeignorePresent_IsInfrastructureFailure_FailsClosed() + { + var scanExecs = 0; + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec)) + return Task.FromResult(Ok(exec)); + if (IsIgnoreProbe(exec)) + // exit 0 = .lycheeignore exists at the repo root + return Task.FromResult(new SandboxExecResult(0, "", "")); + scanExecs++; + return Task.FromResult(new SandboxExecResult(0, JsonClean, "")); + }); + + IAuditor auditor = new LycheeAuditor(); + var ex = await Assert.ThrowsAsync( + () => auditor.RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None)); + + Assert.Contains(".lycheeignore", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, scanExecs); + } + + [Fact] + public async Task TrustRepositorySuppression_SkipsIgnoreGate_AndDropsConfigPin() + { + SandboxExec? scanExec = null; + var ignoreProbes = 0; + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec)) + return Task.FromResult(Ok(exec)); + if (IsIgnoreProbe(exec)) + { + ignoreProbes++; + return Task.FromResult(new SandboxExecResult(0, "", "")); + } + scanExec = exec; + return Task.FromResult(new SandboxExecResult(0, JsonClean, "")); + }); + + var auditor = new LycheeAuditor(); + await auditor.InitializeAsync( + BuildPluginContext(new Dictionary + { + ["Scoped:TrustRepositorySuppression"] = "true", + }), + CancellationToken.None); + + var result = await ((IAuditor)auditor).RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None); + + Assert.True(result.Passed); + Assert.Equal(0, ignoreProbes); + Assert.NotNull(scanExec); + Assert.DoesNotContain("--config", scanExec!.Argv); + } + + [Fact] + public async Task ScopedConfiguration_ConfigPath_OverridesPinnedConfig() + { + SandboxExec? scanExec = null; + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec) || IsIgnoreProbe(exec)) + return Task.FromResult(Ok(exec)); + scanExec = exec; + return Task.FromResult(new SandboxExecResult(0, JsonClean, "")); + }); + + var auditor = new LycheeAuditor(); + await auditor.InitializeAsync( + BuildPluginContext(new Dictionary + { + ["Scoped:ConfigPath"] = "/opt/codeybox/lychee.toml", + }), + CancellationToken.None); + + await ((IAuditor)auditor).RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None); + + Assert.NotNull(scanExec); + var argv = scanExec!.Argv; + var configIndex = argv.ToList().IndexOf("--config"); + Assert.True(configIndex >= 0 && configIndex + 1 < argv.Count); + Assert.Equal("/opt/codeybox/lychee.toml", argv[configIndex + 1]); + Assert.DoesNotContain("/dev/null", argv); + } + + [Fact] + public async Task ScopedConfiguration_Inputs_ReplaceDefaultWholeTreeScope() + { + SandboxExec? scanExec = null; + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec) || IsIgnoreProbe(exec)) + return Task.FromResult(Ok(exec)); + scanExec = exec; + return Task.FromResult(new SandboxExecResult(0, JsonClean, "")); + }); + + var auditor = new LycheeAuditor(); + await auditor.InitializeAsync( + BuildPluginContext(new Dictionary + { + ["Scoped:Inputs"] = "docs, README.md", + }), + CancellationToken.None); + + await ((IAuditor)auditor).RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None); + + Assert.NotNull(scanExec); + Assert.Equal("docs", scanExec!.Argv[^2]); + Assert.Equal("README.md", scanExec.Argv[^1]); + Assert.DoesNotContain(".", scanExec.Argv); + } + + [Fact] + public async Task ScopedConfiguration_CheckRemoteLinks_DropsOffline_AndRequiresNetwork() + { + SandboxExec? scanExec = null; + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec) || IsIgnoreProbe(exec)) + return Task.FromResult(Ok(exec)); + scanExec = exec; + return Task.FromResult(new SandboxExecResult(0, JsonClean, "")); + }); + + var auditor = new LycheeAuditor(); + Assert.Equal(AuditCapabilities.None, ((IAuditor)auditor).Required); + + await auditor.InitializeAsync( + BuildPluginContext(new Dictionary + { + ["Scoped:CheckRemoteLinks"] = "true", + }), + CancellationToken.None); + + Assert.True(((IAuditor)auditor).Required.HasFlag(AuditCapabilities.Network)); + + await ((IAuditor)auditor).RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None); + + Assert.NotNull(scanExec); + Assert.DoesNotContain("--offline", scanExec!.Argv); + } + + [Fact] + public async Task ScopedConfiguration_ExcludePaths_FiltersVendoredFindings() + { + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec) || IsIgnoreProbe(exec)) + return Task.FromResult(Ok(exec)); + return Task.FromResult(new SandboxExecResult(2, JsonWithFilteredPaths, "")); + }); + + IAuditor auditor = new LycheeAuditor(); + var result = await auditor.RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None); + + // vendor/ and node_modules/ findings are dropped by the default + // ExcludePaths; the docs/ broken link survives. + var finding = Assert.Single(result.Findings); + Assert.Equal("docs/guide.md:7", finding.Location); + Assert.Equal(AuditSeverity.Error, finding.Severity); + } + + [Theory] + [InlineData("vendor/", @"^(?:\./)?vendor/")] + [InlineData("foo.md", @"^(?:\./)?foo\.md$")] + [InlineData("", null)] + [InlineData(null, null)] + public void ExcludePathPattern_TranslatesEntry_ToAnchoredRegex( + string? entry, string? expected) + { + Assert.Equal(expected, LycheeAuditor.ToExcludePathPattern(entry)); + } + + [Fact] + public void ExcludePathPattern_MatchesWalkerPaths_LikeBasePrefixSemantics() + { + var dirPattern = new Regex(LycheeAuditor.ToExcludePathPattern("vendor/")!); + Assert.Matches(dirPattern, "./vendor/lib/README.md"); + Assert.Matches(dirPattern, "vendor/lib/README.md"); + Assert.DoesNotMatch(dirPattern, "./docs/vendor/lib.md"); + + var filePattern = new Regex(LycheeAuditor.ToExcludePathPattern("CONTRIBUTING.md")!); + Assert.Matches(filePattern, "./CONTRIBUTING.md"); + Assert.DoesNotMatch(filePattern, "./docs/CONTRIBUTING.md"); + + // Metacharacters in config entries are data, not pattern text. + var dotted = new Regex(LycheeAuditor.ToExcludePathPattern("a+b/")!); + Assert.Matches(dotted, "./a+b/x.md"); + Assert.DoesNotMatch(dotted, "./ab/x.md"); + } + + [Fact] + public void DisabledPlugin_IsNotLoaded_AndToolAbsentFromBaselineProvisioning() + { + var assemblyPath = PluginAssemblyPath(); + var loader = new PluginLoader( + new PluginOptions + { + AssemblyPaths = [assemblyPath], + Allowlist = ["*"], + Enabled = [], + }, + new ConfigurationBuilder().Build(), + NullLogger.Instance); + + Assert.Empty(loader.DiscoverPlugins()); + var status = Assert.Single( + loader.GetDiscoveryStatuses(), + s => s.PluginId == LycheeAuditor.PluginId); + Assert.Equal(PluginSkipReason.Disabled, status.SkipReason); + + var tools = loader.GetEnabledPluginTools(); + Assert.Empty(tools); + + var contributions = PluginBaselineProvisioning.BuildContributions(tools); + var flattened = string.Join("\n", contributions.InstallCommands) + + "\n" + string.Join("\n", contributions.VerificationCommands.SelectMany(static v => v.Argv)); + Assert.DoesNotContain("lychee", flattened, StringComparison.Ordinal); + } + + [Fact] + public void EnabledPlugin_DeclaresLycheeRequirement_VerifyOnly() + { + var assemblyPath = PluginAssemblyPath(); + var loader = new PluginLoader( + new PluginOptions + { + AssemblyPaths = [assemblyPath], + Allowlist = ["*"], + Enabled = [LycheeAuditor.PluginId], + }, + new ConfigurationBuilder().Build(), + NullLogger.Instance); + + var plugins = loader.DiscoverPlugins(); + Assert.Contains(plugins, p => p.PluginId == LycheeAuditor.PluginId); + + var tool = Assert.Single(loader.GetEnabledPluginTools()); + Assert.Equal("lychee", tool.Binary); + // Verify-only by design: no distro apt package carries a version pin. + Assert.Null(tool.AptPackage); + + var contributions = PluginBaselineProvisioning.BuildContributions(loader.GetEnabledPluginTools()); + var verification = Assert.Single(contributions.VerificationCommands); + Assert.Contains("lychee", string.Join(" ", verification.Argv), StringComparison.Ordinal); + Assert.Empty(contributions.InstallCommands); + } + + [Fact] + [Trait("requires_lychee", "true")] + public async Task RealLychee_BrokenLocalLink_YieldsFinding() + { + var installed = InstalledLycheeVersion; + if (installed is null) + return; + + var fixtureDir = await SeedLycheeFixtureRepoAsync(clean: false); + + try + { + var provider = new ProcessSandboxProvider(NullLogger.Instance); + await using var sandbox = await provider.CreateAsync( + new SandboxSpec + { + ImageReference = "ignored", + WorkingDirectory = "/work", + Mounts = [new SandboxMount { SandboxPath = "/work", HostPath = fixtureDir }], + }, + CancellationToken.None); + + var auditor = new LycheeAuditor(); + await auditor.InitializeAsync( + BuildPluginContext(new Dictionary + { + ["Scoped:ExpectedVersion"] = installed, + }), + CancellationToken.None); + + var result = await ((IAuditor)auditor).RunAsync( + sandbox, "/work", FakeContext(), CancellationToken.None); + + Assert.False(result.Passed); + var finding = Assert.Single(result.Findings, f => f.Title.Contains("missing-target.md", StringComparison.Ordinal)); + Assert.Equal(AuditSeverity.Error, finding.Severity); + Assert.Equal("bad.md:1", finding.Location); + } + finally + { + TryDeleteDirectory(fixtureDir); + } + } + + [Fact] + [Trait("requires_lychee", "true")] + public async Task RealLychee_CleanFixture_Passes() + { + var installed = InstalledLycheeVersion; + if (installed is null) + return; + + var fixtureDir = await SeedLycheeFixtureRepoAsync(clean: true); + + try + { + var provider = new ProcessSandboxProvider(NullLogger.Instance); + await using var sandbox = await provider.CreateAsync( + new SandboxSpec + { + ImageReference = "ignored", + WorkingDirectory = "/work", + Mounts = [new SandboxMount { SandboxPath = "/work", HostPath = fixtureDir }], + }, + CancellationToken.None); + + var auditor = new LycheeAuditor(); + await auditor.InitializeAsync( + BuildPluginContext(new Dictionary + { + ["Scoped:ExpectedVersion"] = installed, + }), + CancellationToken.None); + + var result = await ((IAuditor)auditor).RunAsync( + sandbox, "/work", FakeContext(), CancellationToken.None); + + Assert.True(result.Passed); + Assert.Empty(result.Findings); + } + finally + { + TryDeleteDirectory(fixtureDir); + } + } + + private static string PluginAssemblyPath() + { + var path = Path.Combine(AppContext.BaseDirectory, "CodeyBox.LycheeAuditorPlugin.dll"); + Assert.True(File.Exists(path), $"Plugin assembly not found at '{path}'."); + return path; + } + + private static PluginContext BuildPluginContext(IReadOnlyDictionary scopedValues) + { + var config = new ConfigurationBuilder() + .AddInMemoryCollection(scopedValues) + .Build(); + return new PluginContext( + HostApiVersion: "1.0", + PluginId: LycheeAuditor.PluginId, + PluginDisplayName: "CodeyBox: Lychee Broken Link Checker", + Host: new TestPluginHost(config.GetSection("Scoped"))); + } + + private static SandboxExecResult Ok(SandboxExec exec) + => IsVersionProbe(exec) + ? new SandboxExecResult(0, "lychee " + LycheeAuditor.DefaultExpectedVersion + "\n", "") + : IsIgnoreProbe(exec) + // exit 1 = .lycheeignore absent + ? new SandboxExecResult(1, "", "") + : new SandboxExecResult(0, "", ""); + + private static bool IsPresenceProbe(SandboxExec exec) + => exec.Argv.Count >= 3 + && exec.Argv[0] == "sh" + && exec.Argv[1] == "-c" + && exec.Argv[2].Contains("command -v", StringComparison.Ordinal) + && exec.Argv.Contains("lychee", StringComparer.Ordinal); + + private static bool IsVersionProbe(SandboxExec exec) + => exec.Argv.Count == 2 && exec.Argv[0] == "lychee" && exec.Argv[1] == "--version"; + + private static bool IsIgnoreProbe(SandboxExec exec) + => exec.Argv.Count >= 3 + && exec.Argv[0] == "sh" + && exec.Argv[1] == "-c" + && exec.Argv.Contains(".lycheeignore", StringComparer.Ordinal); + + private static async Task SeedLycheeFixtureRepoAsync(bool clean) + { + var dir = Path.Combine( + Path.GetTempPath(), "codeybox-lychee-fixture-" + Guid.NewGuid().ToString("N")[..8]); + Directory.CreateDirectory(dir); + + if (clean) + { + await File.WriteAllTextAsync( + Path.Combine(dir, "ok.md"), "[existing](./target.md)\n"); + await File.WriteAllTextAsync(Path.Combine(dir, "target.md"), "# Target\n"); + } + else + { + await File.WriteAllTextAsync( + Path.Combine(dir, "bad.md"), "[missing](./missing-target.md)\n"); + } + + return dir; + } + + private static string? ProbeInstalledLycheeVersion() + { + try + { + var psi = new ProcessStartInfo + { + FileName = "lychee", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + psi.ArgumentList.Add("--version"); + using var process = Process.Start(psi)!; + var stdout = process.StandardOutput.ReadToEnd(); + if (!process.WaitForExit(milliseconds: 10_000)) + { + try { process.Kill(); } catch { /* best-effort probe teardown */ } + return null; + } + var match = Regex.Match(stdout, @"\d+\.\d+\.\d+[\w.\-]*"); + return process.ExitCode == 0 && match.Success ? match.Value : null; + } + catch + { + return null; + } + } + + private static void TryDeleteDirectory(string path) + { + try { Directory.Delete(path, recursive: true); } + catch { /* best-effort fixture cleanup */ } + } + + private static AuditContext FakeContext() => + new(WorkItemId.New(), "feature", "main", 1, "do x"); + + private sealed class TestPluginHost(IConfigurationSection scoped) : IPluginHost + { + public Microsoft.Extensions.Logging.ILogger Logger { get; } = NullLogger.Instance; + public IConfigurationSection ScopedConfig { get; } = scoped; + } + + private sealed class FakeSandbox( + Func> onExec) : ISandbox + { + public string Id => "fake"; + + public async Task ExecAsync(SandboxExec exec, CancellationToken ct = default) + { + await Task.Yield(); + ct.ThrowIfCancellationRequested(); + return await onExec(exec, ct); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} From 0598b84805a349b03ad38e4dc11268b7d67175bc Mon Sep 17 00:00:00 2001 From: Adam Frisby Date: Fri, 25 Sep 2026 18:38:12 +0000 Subject: [PATCH 2/2] Close walker-suppression channel in lychee audit; share precondition probes in PluginSdk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses audit rework findings for the lychee auditor plugin: - The scan now passes --no-ignore by default: lychee's walker previously honored .gitignore/.ignore/global ignore files, so the audit subject could commit a doc file and exclude it from the crawl — a silent false pass. The .lycheeignore presence gate stays (it is a URL-exclusion file untouched by --no-ignore) and --config /dev/null still pins out repo config files. TrustRepositorySuppression opts back in to all three surfaces. The ExcludePaths defaults widen (bin/, obj/, target/, .venv/, venv/) since gitignored build output is now walked. - The repo-file presence gate is now a shared fail-closed helper on ExternalToolAuditorBase (ProbeRepositoryFilesPresentAsync): the probe echoes present paths on stdout and any non-zero exit is "could not confirm absence" -> AuditUnavailableException, ending the divergence where lychee's fork treated every non-zero exit as "absent". Gitleaks uses the same probe; its git-history check stays gitleaks-specific. - The pinned-version precondition is now declared, not implemented: plugins set VersionPin (new ToolVersionPin record carrying plugin id, a per-invocation ExpectedVersion accessor, the default release, and the probe arguments) and RunAsync enforces it before VerifyToolAsync. This removes the fourth copy of the version-probe scaffolding across gitleaks/eslint/spectral/lychee. The ExpectedVersion scoped key is now the named constant ToolVersionPin.ExpectedVersionKey. - Operator flag detection in ExtraArguments is shared (ExtraArgumentsSupplyFlag) and matches --flag, --flag=value, and the joined short form (-cfoo), so an operator --config=path or --root-dir=path is honored instead of producing a duplicate flag. ExcludePaths normalization is shared (NormalizeExcludePathEntry) so the findings filter and --exclude-path translation cannot drift, and ExternalToolAuditorOptions.SplitCommaSeparatedList replaces the private SplitList copies. - Tests: ignore-probe fakes follow the new convention (exit 0 + names = present); new coverage for probe-error fail-closed, --no-ignore argv, attached-form config overrides, and the shared helpers' path validation; ProbeInstalledLycheeVersion now drains stdout and stderr concurrently (previously a full stderr pipe could deadlock ahead of the timeout); the vacuous baseline-provisioning assertion is removed. Not done: a "--" separator before positional Inputs is deliberately omitted — the base appends operator ExtraArguments after the built argv, so a mid-argv "--" would reclassify those extras as inputs. Inputs is operator configuration, and a leading-dash input fails closed as a lychee usage error (infrastructure), not a pass. CodeyBox-Prompt-Revision: 1 Co-Authored-By: CodeyBox --- docs/extending/auditor-plugins.md | 17 +- .../LycheeAuditor.cs | 256 +++++----------- .../CodeyBox.LycheeAuditorPlugin/README.md | 55 ++-- .../EslintAuditor.cs | 119 +------- .../SpectralAuditor.cs | 126 +------- .../GitleaksAuditor.cs | 163 ++-------- .../Tools/ExternalToolAuditorBase.cs | 282 +++++++++++++++++- .../Tools/ExternalToolAuditorOptions.cs | 23 +- .../Tools/ToolVersionPin.cs | 43 +++ .../ExternalToolAuditorTests.cs | 78 +++++ tests/CodeyBox.Tests/GitleaksAuditorTests.cs | 6 +- tests/CodeyBox.Tests/LycheeAuditorTests.cs | 86 +++++- 12 files changed, 657 insertions(+), 597 deletions(-) create mode 100644 src/CodeyBox.PluginSdk/Tools/ToolVersionPin.cs diff --git a/docs/extending/auditor-plugins.md b/docs/extending/auditor-plugins.md index 2adf95ca6..f26881827 100644 --- a/docs/extending/auditor-plugins.md +++ b/docs/extending/auditor-plugins.md @@ -308,14 +308,23 @@ Behaviour the base guarantees identically for every tool: the plugin's scoped config section; resolve it per invocation so edits apply without a restart. +A pinned tool version is declared, not implemented: set `VersionPin` to a +`ToolVersionPin` (plugin id, a per-invocation `ExpectedVersion` accessor, the +default release, and the version-probe arguments — `["--version"]` for most +tools, `["version"]` for subcommand-style CLIs) and `RunAsync` probes the +tool before every scan, failing closed on a missing binary, an unrecognised +version, or a mismatch. + Two extension points cover tool requirements the base cannot express declaratively: - `VerifyToolAsync` — a pre-scan precondition hook invoked inside `RunAsync` - after the tool's presence is confirmed. Override it for checks like a pinned - tool version or repository-state gates, throwing `AuditUnavailableException` - to fail closed. Run probes through `ExecToolBoundedAsync` so they inherit - the same timeout bounding and failure classification as the scan. + after the tool's presence and declared `VersionPin` are confirmed. Override + it for repository-state gates (e.g. refusing repo-authored suppression + files — `ProbeRepositoryFilesPresentAsync` is the shared fail-closed probe + for that), throwing `AuditUnavailableException` to fail closed. Run probes + through `ExecToolBoundedAsync` so they inherit the same timeout bounding + and failure classification as the scan. - `BuildToolEnvironment` — extra environment variables for the tool process, for tools whose behavior is env-controlled (e.g. pinning configuration that must not come from the audited repository). diff --git a/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/LycheeAuditor.cs b/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/LycheeAuditor.cs index 6f195666f..2abbb2e5d 100644 --- a/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/LycheeAuditor.cs +++ b/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/LycheeAuditor.cs @@ -12,10 +12,11 @@ namespace CodeyBox.LycheeAuditorPlugin; /// invocation with a bounded timeout, per-stream output caps, exit-code /// classification, severity mapping, finding identity, and per-auditor /// configuration. This class adds the lychee JSON report parser -/// (), the pinned tool-version probe and -/// the repository-suppression gate via -/// , and the offline/remote -/// posture below. +/// (), the pinned tool version via +/// , the +/// repository-suppression gate via +/// , and the +/// offline/remote posture below. /// /// Gate behaviour: hybrid / severity-driven. Confirmed link /// failures (error_map — dead URLs, missing files, missing fragments) @@ -58,38 +59,43 @@ namespace CodeyBox.LycheeAuditorPlugin; /// project's audit-tool network profile). Findings in remote mode are /// inherently non-deterministic. /// -/// Repository-controlled suppression. lychee honors surfaces -/// authored inside the audited repository: a .lycheeignore file in the -/// working directory is loaded unconditionally (no flag disables it) and its -/// lines become URL-exclusion regexes, and when no --config is given -/// lychee auto-loads configuration from a repo-root lychee.toml (or a +/// Repository-controlled suppression. The audit subject writes +/// the repository, and lychee honors surfaces authored inside it: the walker +/// skips input files matched by .gitignore, .ignore, or the +/// global ignore file; a .lycheeignore in the working directory is +/// loaded unconditionally (no flag disables it) and its lines become +/// URL-exclusion regexes; and when no --config is given lychee +/// auto-loads configuration from a repo-root lychee.toml (or a /// [lychee] section in Cargo.toml/pyproject.toml/ -/// package.json). The audit subject writes that repository, so by -/// default a .lycheeignore at the repo root fails closed as -/// infrastructure and the scan pins configuration with -/// --config /dev/null, which disables every default config-file lookup -/// (an operator ConfigPath or --config in ExtraArguments -/// takes precedence over the pin). TrustRepositorySuppression opts in -/// to the repository's own suppression surfaces. Note lychee also skips input -/// files matched by .gitignore/.ignore rules; the auditor keeps -/// that default because --no-ignore would crawl gitignored build -/// output present in the worktree. Residual gap, documented not hidden: a -/// subject could commit a file and then exclude it via .gitignore so -/// lychee's walker skips it — matching a dedicated suppression file's effect -/// through a file nearly every repo legitimately has, which is why it is not -/// gated like .lycheeignore. +/// package.json). By default the scan neutralizes all three channels: +/// it passes --no-ignore so ignore rules cannot hide committed +/// documentation from the crawl, fails closed as infrastructure when a +/// repo-root .lycheeignore exists (checked through the shared +/// fail-closed file-presence probe), and pins configuration with +/// --config /dev/null, which disables every default config-file +/// lookup (an operator ConfigPath or --config in +/// ExtraArguments takes precedence over the pin). +/// TrustRepositorySuppression opts back in to the repository's own +/// suppression surfaces — it drops --no-ignore and the +/// .lycheeignore gate and lets lychee's default config lookup run. +/// Because --no-ignore also walks gitignored build output, the +/// vendored/generated ExcludePaths defaults double as the crawl +/// boundary via --exclude-path. /// /// Scope and defaults. The scan is lychee .: the worktree /// is walked recursively, filtered to lychee's documentation extensions /// (Markdown, HTML, CSS, XML, plaintext). Hidden files are included /// (--hidden) because documentation legitimately lives under /// dot-directories such as .github/; .git/ is excluded. Links -/// under vendored (vendor/, third_party/, node_modules/) -/// and generated (dist/, build/, out/, coverage/) +/// under vendored (vendor/, third_party/, node_modules/, +/// .venv/, venv/) and generated (dist/, build/, +/// out/, coverage/, bin/, obj/, target/) /// prefixes are excluded at extraction time (--exclude-path, derived /// from ExcludePaths) and findings under them are dropped as well: /// problems there belong to upstream packages or build output, not the -/// change under audit. --include-fragments checks #anchor +/// change under audit — a boundary that matters more under +/// --no-ignore, which no longer lets .gitignore keep such +/// trees out of the walk. --include-fragments checks #anchor /// targets inside local files — a common real-world docs break. Root-absolute /// links (/docs/x.md) resolve against RootDirectory when /// configured; otherwise lychee flags them, which is the honest result — the @@ -145,7 +151,8 @@ public sealed class LycheeAuditor : ExternalToolAuditorBase, IPluginInitializer /// /// Scoped-config key opting in to repository-authored suppression — - /// .lycheeignore and the default lychee config files + /// walker ignore files (.gitignore/.ignore), + /// .lycheeignore, and the default lychee config files /// (lychee.toml and friends). Default false: the audited repo must /// not be able to silence the audit. /// @@ -169,16 +176,7 @@ public sealed class LycheeAuditor : ExternalToolAuditorBase, IPluginInitializer // flag to redirect or disable it, so presence at the repo root is gated. private const string RepositoryIgnoreFile = ".lycheeignore"; - private const int ProbeMaxOutputBytes = 16 * 1024; - private const int MessageValueMaxChars = 64; - // The precondition probes are liveness checks, not the scan: they never - // need more than this and share the operator-configured timeout below it. - private static readonly TimeSpan ProbeTimeoutCap = TimeSpan.FromSeconds(30); - private static readonly Regex VersionPattern = new( - @"\d+\.\d+\.\d+[\w.\-]*", - RegexOptions.CultureInvariant | RegexOptions.Compiled); - - private static readonly ExternalToolAuditorOptions AuditorDefaults = new() + private static ExternalToolAuditorOptions CreateDefaults() => new() { // 0 = all checked links OK; 2 = link check failures. Both emit the // JSON report — both are verdicts. 1 (missing input / runtime / @@ -188,23 +186,29 @@ public sealed class LycheeAuditor : ExternalToolAuditorBase, IPluginInitializer // Findings in VCS internals, vendored/dependency trees, and generated // build output do not describe the change under audit — noise that // trains operators to ignore the auditor. Each entry is also passed - // to lychee as --exclude-path so those files are never crawled. - // Operators re-include a path by overriding ExcludePaths in scoped - // config. + // to lychee as --exclude-path so those files are never crawled; under + // --no-ignore this list is the only thing keeping gitignored build + // output out of the walk. Operators re-include a path by overriding + // ExcludePaths in scoped config. ExcludePaths = [ ".git/", "vendor/", "third_party/", "node_modules/", + ".venv/", + "venv/", "dist/", "build/", "out/", "coverage/", + "bin/", + "obj/", + "target/", ], }; - private Func _optionsAccessor = () => AuditorDefaults; + private Func _optionsAccessor = CreateDefaults; private Func _expectedVersion = static () => DefaultExpectedVersion; private Func _configPath = static () => null; private Func _rootDirectory = static () => null; @@ -250,9 +254,16 @@ public sealed class LycheeAuditor : ExternalToolAuditorBase, IPluginInitializer /// protected override Func OptionsAccessor => _optionsAccessor; + /// + protected override ToolVersionPin? VersionPin => + new(PluginId, _expectedVersion, DefaultExpectedVersion, ["--version"]); + /// protected override IReadOnlyList BuildToolArguments(ExternalToolAuditorOptions options) { + var operatorSuppliesConfig = ExtraArgumentsSupplyFlag(options, "--config", "-c"); + var operatorSuppliesRootDir = ExtraArgumentsSupplyFlag(options, "--root-dir"); + var args = new List { // The stats report on stdout is the verdict; the parser reads @@ -272,28 +283,32 @@ protected override IReadOnlyList BuildToolArguments(ExternalToolAuditorO if (!_checkRemoteLinks()) args.Add("--offline"); + // Repository-authored walker suppression (.gitignore, .ignore, the + // global ignore file) would let the audit subject hide a documented + // broken link from the crawl — the subject writes those files. Unless + // the operator opts in to repository suppression, the walker ignores + // them; the ExcludePaths-derived --exclude-path flags below keep + // vendored/generated trees out of the crawl instead. + if (!_trustRepositorySuppression()) + args.Add("--no-ignore"); + // Config precedence: explicit operator config, else the pinned empty // config that disables repo config loading, else (when trusting the // repository) lychee's own default lookup. var configPath = _configPath(); - if (!string.IsNullOrWhiteSpace(configPath) - && !options.ExtraArguments.Contains("--config", StringComparer.Ordinal) - && !options.ExtraArguments.Contains("-c", StringComparer.Ordinal)) + if (!string.IsNullOrWhiteSpace(configPath) && !operatorSuppliesConfig) { args.Add("--config"); args.Add(configPath.Trim()); } - else if (!_trustRepositorySuppression() - && !options.ExtraArguments.Contains("--config", StringComparer.Ordinal) - && !options.ExtraArguments.Contains("-c", StringComparer.Ordinal)) + else if (!_trustRepositorySuppression() && !operatorSuppliesConfig) { args.Add("--config"); args.Add(PinnedEmptyConfigPath); } var rootDirectory = _rootDirectory(); - if (!string.IsNullOrWhiteSpace(rootDirectory) - && !options.ExtraArguments.Contains("--root-dir", StringComparer.Ordinal)) + if (!string.IsNullOrWhiteSpace(rootDirectory) && !operatorSuppliesRootDir) { args.Add("--root-dir"); args.Add(rootDirectory.Trim()); @@ -326,11 +341,11 @@ public Task InitializeAsync(PluginContext context, CancellationToken ct = defaul { ArgumentNullException.ThrowIfNull(context); var scoped = context.ScopedConfig; - _optionsAccessor = () => ExternalToolAuditorOptions.Bind(scoped, AuditorDefaults); - _expectedVersion = () => scoped["ExpectedVersion"]; + _optionsAccessor = () => ExternalToolAuditorOptions.Bind(scoped, CreateDefaults()); + _expectedVersion = () => scoped[ToolVersionPin.ExpectedVersionKey]; _configPath = () => scoped[ConfigPathKey]; _rootDirectory = () => scoped[RootDirectoryKey]; - _inputs = () => SplitList(scoped[InputsKey]); + _inputs = () => ExternalToolAuditorOptions.SplitCommaSeparatedList(scoped[InputsKey]); _checkRemoteLinks = () => bool.TryParse(scoped[CheckRemoteLinksKey], out var check) && check; _trustRepositorySuppression = () => @@ -341,10 +356,10 @@ public Task InitializeAsync(PluginContext context, CancellationToken ct = defaul } /// - /// lychee-specific preconditions on the live path: the pinned scanner - /// version, and — unless the operator opted in — absence of a repo-root - /// .lycheeignore, which lychee would load unconditionally. Both - /// fail closed as infrastructure before the scan runs. + /// lychee-specific precondition on the live path beyond the base's pinned + /// version check: unless the operator opted in, a repo-root + /// .lycheeignore — which lychee would load unconditionally — fails + /// closed as infrastructure before the scan runs. /// protected override async Task VerifyToolAsync( ISandbox sandbox, @@ -353,63 +368,11 @@ protected override async Task VerifyToolAsync( ExternalToolAuditorOptions options, CancellationToken ct) { - await ThrowIfToolVersionMismatchAsync(sandbox, workingDirectory, tool, options, ct) - .ConfigureAwait(false); if (!_trustRepositorySuppression()) await ThrowIfRepositoryIgnoreFilePresentAsync(sandbox, workingDirectory, tool, options, ct) .ConfigureAwait(false); } - private async Task ThrowIfToolVersionMismatchAsync( - ISandbox sandbox, - string workingDirectory, - string tool, - ExternalToolAuditorOptions options, - CancellationToken ct) - { - var configured = _expectedVersion(); - var expected = NormalizeVersion(configured); - if (expected is null) - throw new AuditUnavailableException( - $"could-not-verify: auditor '{Name}' has an unparseable ExpectedVersion " - + $"('{TruncateForMessage(configured)}'); set CodeyBox:Plugins:{PluginId}:ExpectedVersion " - + $"to a {tool} release such as '{DefaultExpectedVersion}'.") - { IsDeterministic = true }; - - var result = await ExecToolBoundedAsync( - sandbox, - tool, - "version check", - new SandboxExec - { - Argv = [tool, "--version"], - WorkingDirectory = workingDirectory, - MaxStdoutBytes = ProbeMaxOutputBytes, - MaxStderrBytes = ProbeMaxOutputBytes, - KillOnOutputLimit = true, - }, - ProbeTimeout(options), - ct).ConfigureAwait(false); - - var reported = ExtractVersion(result.Stdout); - if (result.ExecutionUnavailable - || result.ExitCode != 0 - || reported is null) - throw new AuditUnavailableException( - $"could-not-verify: audit tool '{tool}' version could not be determined " - + $"(exit {result.ExitCode}). The pinned release is required before the scan can run — " - + $"a missing or foreign '{tool}' is infrastructure, not a verdict on the diff.", - result.ExitCode, - result.Stdout + "\n" + result.Stderr); - - if (!string.Equals(reported, expected, StringComparison.Ordinal)) - throw new AuditUnavailableException( - $"could-not-verify: audit tool '{tool}' is version {reported}, but this auditor is " - + $"pinned to {expected}. A different checker version changes the link checks and the " - + "findings; provision the pinned release or set ExpectedVersion to the version you provisioned.") - { IsDeterministic = true }; - } - private async Task ThrowIfRepositoryIgnoreFilePresentAsync( ISandbox sandbox, string workingDirectory, @@ -420,36 +383,22 @@ private async Task ThrowIfRepositoryIgnoreFilePresentAsync( // lychee loads /.lycheeignore unconditionally — its lines become // URL-exclusion regexes the audit subject could use to hide a broken // link — and no flag redirects or disables that load, so presence is - // gated rather than outranked. (-L covers a dangling symlink that -e - // would miss.) - var presence = await ExecToolBoundedAsync( + // gated rather than outranked. (.gitignore/.ignore need no gate: the + // scan runs --no-ignore so they cannot suppress input files.) + var present = await ProbeRepositoryFilesPresentAsync( sandbox, + workingDirectory, tool, - "suppression check", - new SandboxExec - { - Argv = ["sh", "-c", "[ -e \"$1\" ] || [ -L \"$1\" ]", "sh", RepositoryIgnoreFile], - WorkingDirectory = workingDirectory, - MaxStdoutBytes = ProbeMaxOutputBytes, - MaxStderrBytes = ProbeMaxOutputBytes, - KillOnOutputLimit = true, - }, - ProbeTimeout(options), + [RepositoryIgnoreFile], + options, ct).ConfigureAwait(false); - - if (presence.ExecutionUnavailable) - throw new AuditUnavailableException( - $"could-not-verify: audit tool '{tool}' suppression check could not run: the sandbox exec " - + "transport was unavailable."); - if (presence.ExitCode == 0) + if (present.Count > 0) throw new AuditUnavailableException( $"could-not-verify: audit tool '{tool}' found '{RepositoryIgnoreFile}' at the root of the " + "audited repository — lychee loads it unconditionally and honors its URL-exclusion " + "patterns, so the audit subject could use it to hide a broken link. Remove the file, " - + "or set CodeyBox:Plugins:" + PluginId + ":" + TrustRepositorySuppressionKey - + " to true to trust repository-controlled suppression surfaces.", - presence.ExitCode, - presence.Stdout + "\n" + presence.Stderr) + + $"or set CodeyBox:Plugins:{PluginId}:{TrustRepositorySuppressionKey}" + + " to true to trust repository-controlled suppression surfaces.") { IsDeterministic = true }; } @@ -465,52 +414,11 @@ private async Task ThrowIfRepositoryIgnoreFilePresentAsync( /// internal static string? ToExcludePathPattern(string? entry) { - if (string.IsNullOrWhiteSpace(entry)) - return null; - var normalized = entry.Replace('\\', '/').Trim().TrimStart('/'); - if (normalized.Length == 0) + var normalized = NormalizeExcludePathEntry(entry); + if (normalized is null) return null; return normalized.EndsWith('/') ? @"^(?:\./)?" + Regex.Escape(normalized) : @"^(?:\./)?" + Regex.Escape(normalized) + "$"; } - - private static TimeSpan ProbeTimeout(ExternalToolAuditorOptions options) - { - var timeout = EffectiveTimeout(options); - return timeout > ProbeTimeoutCap ? ProbeTimeoutCap : timeout; - } - - private static string? ExtractVersion(string stdout) - { - // `lychee --version` prints "lychee 0.24.2". - var match = VersionPattern.Match(stdout); - return match.Success ? match.Value : null; - } - - private static string? NormalizeVersion(string? configured) - { - var value = string.IsNullOrWhiteSpace(configured) - ? DefaultExpectedVersion - : configured.Trim(); - var match = VersionPattern.Match(value); - return match.Success ? match.Value : null; - } - - private static string TruncateForMessage(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - return "(empty)"; - var single = SingleLine(value); - return single.Length > MessageValueMaxChars - ? single[..MessageValueMaxChars] + "…" - : single; - } - - private static List SplitList(string? value) - => string.IsNullOrWhiteSpace(value) - ? [] - : value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - .Where(static item => item.Length > 0) - .ToList(); } diff --git a/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/README.md b/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/README.md index ec9c063bb..d4d127969 100644 --- a/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/README.md +++ b/plugins/auditors-documentation/CodeyBox.LycheeAuditorPlugin/README.md @@ -2,8 +2,8 @@ Auditor plugin wrapping [lychee](https://lychee.cli.rs): it checks links in the audited repository's documentation files with `lychee --format json ---offline .` and reports each failed link check as an audit finding with the -source file and `file:line` location lychee supplies. +--offline --no-ignore .` and reports each failed link check as an audit +finding with the source file and `file:line` location lychee supplies. ## What it reports @@ -42,9 +42,12 @@ source file and `file:line` location lychee supplies. lychee's documentation extensions (`md`/`mkd`/`mdx`/`mdown`/`mdwn`/`mkdn`/ `mkdown`/`markdown`, `html`/`htm`, `css`, `txt`, `xml`). Links inside `*.py`, `*.cs`, `*.yml`, etc. are not extracted. -- **Gitignored and excluded inputs.** lychee skips files covered by - `.gitignore`/`.ignore`, and paths listed in `ExcludePaths` are excluded at - crawl time via `--exclude-path` — links inside them are never checked. +- **Excluded inputs.** Paths listed in `ExcludePaths` are excluded at crawl + time via `--exclude-path` — links inside them are never checked. Walker + ignore files (`./.gitignore`, `.ignore`, the global ignore file) are *not* + honored by default: the scan passes `--no-ignore` so the audit subject + cannot hide a committed file from the crawl. Only + `TrustRepositorySuppression: true` restores ignore-file handling. - **Site-root-absolute links resolve only with `RootDirectory`.** A link like `/docs/guide.md` is only checkable if the deploy root is known; without `RootDirectory` lychee reports it — which is the honest result. @@ -123,27 +126,29 @@ Scoped under `CodeyBox:Plugins:codeybox.lychee`, resolved per run | `RootDirectory` | `null` | Absolute in-sandbox path passed to `--root-dir`; resolves site-root-absolute links (`/docs/x.md`) in local files. Ignored when `ExtraArguments` supplies `--root-dir`. | | `Inputs` | `.` | Comma-separated lychee inputs (files, globs, directories) replacing the whole-tree `.` default — e.g. `docs,README.md` to scope to one tree. | | `ConfigPath` | `null` | Path passed to `--config` — an operator-pinned lychee config. Takes precedence over the `/dev/null` pin and the repo's own config files. Ignored when `ExtraArguments` already supplies `--config`/`-c`. | -| `TrustRepositorySuppression` | `false` | When `false` (default) a repo-root `.lycheeignore` fails closed and `--config /dev/null` disables all default config-file lookup. When `true`, lychee honors `.lycheeignore` and repo-authored `lychee.toml`/`[lychee]` sections. | +| `TrustRepositorySuppression` | `false` | When `false` (default) a repo-root `.lycheeignore` fails closed, `--config /dev/null` disables all default config-file lookup, and `--no-ignore` stops `.gitignore`/`.ignore` from hiding input files. When `true`, lychee honors all of those repo-authored surfaces. | | `MinimumSeverity` | `info` | Drop mapped findings below this severity (`info`, `warning`, `error`). | | `IncludedRules` / `ExcludedRules` | — | Exact rule ids to keep/drop; the only ids are `lychee/broken-link` and `lychee/timeout`. | -| `ExcludePaths` | `.git/`, `vendor/`, `third_party/`, `node_modules/`, `dist/`, `build/`, `out/`, `coverage/` | Repo-relative paths dropped from findings — exact path, or directory prefix when trailing `/`. Each entry is also translated to an anchored `--exclude-path` regex so excluded trees are never crawled. Setting it replaces the default list. | +| `ExcludePaths` | `.git/`, `vendor/`, `third_party/`, `node_modules/`, `.venv/`, `venv/`, `dist/`, `build/`, `out/`, `coverage/`, `bin/`, `obj/`, `target/` | Repo-relative paths dropped from findings — exact path, or directory prefix when trailing `/`. Each entry is also translated to an anchored `--exclude-path` regex so excluded trees are never crawled; under `--no-ignore` this list is the only boundary keeping gitignored build output out of the walk. Setting it replaces the default list. | | `ExtraArguments` | — | Extra argv appended after the built-in args (never via a shell). Useful for `--exclude `, `--accept `, `--scheme https`, or an operator `--config`/`--exclude-path`. A repeated flag wins over the built-in default — take care: `--format` would replace the JSON report the parser expects and break the run into infrastructure failure. | | `TimeoutSeconds` | `300` | Per-run bound. Exceeding it is infrastructure, not a pass. | | `MaxOutputBytesPerStream` / `MaxFindings` | `1 MiB` / `1000` | Output/result caps; overruns are reported as truncation. | **Repository-controlled suppression is off by default.** The audit subject -writes the repository, and lychee honors two repo-authored surfaces: +writes the repository, and lychee honors three repo-authored surfaces: walker +ignore files (`.gitignore`, `.ignore`, the global ignore file — they would +let the subject exclude a committed file from the crawl entirely), `.lycheeignore` in the working directory (loaded unconditionally — no flag -disables it — each line a URL-exclusion regex) and the default config files +disables it — each line a URL-exclusion regex), and the default config files (`lychee.toml`, or `[lychee]`-equivalent sections in `Cargo.toml`, `pyproject.toml`, `package.json`), which can widen `exclude`/`accept` and -silence findings. By default the auditor fails closed when a repo-root -`.lycheeignore` exists, and passes `--config /dev/null` — an empty config -that turns off every default lookup — so the checked ruleset cannot be -steered by the audited tree. `TrustRepositorySuppression: true` restores the -tool's default behavior for repos whose own lychee config is the intended -contract; a deliberate `ConfigPath` overrides the pin without trusting the -repo. +silence findings. By default the auditor passes `--no-ignore` so ignore +rules cannot hide inputs, fails closed when a repo-root `.lycheeignore` +exists, and passes `--config /dev/null` — an empty config that turns off +every default lookup — so the checked ruleset cannot be steered by the +audited tree. `TrustRepositorySuppression: true` restores the tool's default +behavior for repos whose own lychee config is the intended contract; a +deliberate `ConfigPath` overrides the pin without trusting the repo. If your repository legitimately ships `lychee.toml` or `.lycheeignore` for its own CI, either point `ConfigPath` at that file (it is then the explicit, @@ -153,10 +158,16 @@ operator-chosen config) or set `TrustRepositorySuppression: true`. `lychee .` — the whole worktree, walked recursively and filtered to documentation extensions, with hidden files included (docs live under -`.github/`) and `.git/` excluded. Findings under vendored -(`vendor/`, `third_party/`, `node_modules/`) and generated (`dist/`, -`build/`, `out/`, `coverage/`) prefixes are excluded both at crawl time -(`--exclude-path`) and at finding level (`ExcludePaths`): broken links in -third-party docs or build output belong to upstream packages, not the change -under audit — reporting them trains operators to ignore the auditor. +`.github/`). The walk does **not** honor `.gitignore`/`.ignore` by default +(`--no-ignore`): a link-checker the audited repository can hide files from +is not a gate. The cost is that gitignored trees are now walked too, so the +`ExcludePaths` defaults — `.git/` plus vendored (`vendor/`, `third_party/`, +`node_modules/`, `.venv/`, `venv/`) and generated (`dist/`, `build/`, `out/`, +`coverage/`, `bin/`, `obj/`, `target/`) prefixes — double as the crawl +boundary: findings under them are excluded both at crawl time +(`--exclude-path`) and at finding level. Exclusions are anchored at the repo +root — a nested `src/X/bin/` is not covered; widen `ExcludePaths` in scoped +config if a layout keeps generated docs elsewhere. Broken links in +third-party docs or build output belong to upstream packages, not the +change under audit — reporting them trains operators to ignore the auditor. `Inputs` narrows the crawl itself when whole-tree coverage is too broad. diff --git a/plugins/auditors-linting/CodeyBox.EslintAuditorPlugin/EslintAuditor.cs b/plugins/auditors-linting/CodeyBox.EslintAuditorPlugin/EslintAuditor.cs index 1203eddab..5323820d1 100644 --- a/plugins/auditors-linting/CodeyBox.EslintAuditorPlugin/EslintAuditor.cs +++ b/plugins/auditors-linting/CodeyBox.EslintAuditorPlugin/EslintAuditor.cs @@ -1,4 +1,3 @@ -using System.Text.RegularExpressions; using CodeyBox.Core; using CodeyBox.PluginSdk; using CodeyBox.PluginSdk.Tools; @@ -16,8 +15,8 @@ namespace CodeyBox.EslintAuditorPlugin; /// --format json-with-metadata, which embeds the process cwd so the /// parser can relativize the absolute filePath values ESLint emits; /// the SARIF formatter is a second unpinned npm package, so the native format -/// is used instead), the pinned tool-version probe via -/// , and the +/// is used instead), the pinned tool-version declaration via +/// , and the /// repository-suppression posture below. /// /// Gate behaviour: hybrid / severity-driven — not blocking on every @@ -112,13 +111,6 @@ public sealed class EslintAuditor : ExternalToolAuditorBase, IPluginInitializer /// internal const string TrustRepositorySuppressionKey = "TrustRepositorySuppression"; - private const int ProbeMaxOutputBytes = 16 * 1024; - private const int MessageValueMaxChars = 64; - private static readonly TimeSpan ProbeTimeoutCap = TimeSpan.FromSeconds(30); - private static readonly Regex VersionPattern = new( - @"\d+\.\d+\.\d+[\w.\-]*", - RegexOptions.CultureInvariant | RegexOptions.Compiled); - private static readonly ExternalToolAuditorOptions AuditorDefaults = new() { // 0 = clean or warnings only; 1 = error-level findings (or a @@ -165,6 +157,10 @@ public sealed class EslintAuditor : ExternalToolAuditorBase, IPluginInitializer /// protected override Func OptionsAccessor => _optionsAccessor; + /// + protected override ToolVersionPin? VersionPin => + new(PluginId, _expectedVersion, DefaultExpectedVersion, ["--version"]); + /// protected override IReadOnlyList BuildToolArguments(ExternalToolAuditorOptions options) { @@ -185,8 +181,7 @@ protected override IReadOnlyList BuildToolArguments(ExternalToolAuditorO var configPath = _configPath(); if (!string.IsNullOrWhiteSpace(configPath) - && !options.ExtraArguments.Contains("--config", StringComparer.Ordinal) - && !options.ExtraArguments.Contains("-c", StringComparer.Ordinal)) + && !ExtraArgumentsSupplyFlag(options, "--config", "-c")) { args.Add("--config"); args.Add(configPath.Trim()); @@ -204,7 +199,7 @@ public Task InitializeAsync(PluginContext context, CancellationToken ct = defaul ArgumentNullException.ThrowIfNull(context); var scoped = context.ScopedConfig; _optionsAccessor = () => ExternalToolAuditorOptions.Bind(scoped, AuditorDefaults); - _expectedVersion = () => scoped["ExpectedVersion"]; + _expectedVersion = () => scoped[ToolVersionPin.ExpectedVersionKey]; _configPath = () => scoped[ConfigPathKey]; _trustRepositorySuppression = () => bool.TryParse(scoped[TrustRepositorySuppressionKey], out var trust) && trust; @@ -212,102 +207,4 @@ public Task InitializeAsync(PluginContext context, CancellationToken ct = defaul "EslintAuditor initialized: pluginId={PluginId}", context.PluginId); return Task.CompletedTask; } - - /// - /// ESLint-specific precondition on the live path: the installed binary - /// must match the pinned release (ExpectedVersion). A mismatch - /// fails closed as infrastructure before the scan runs. - /// - protected override async Task VerifyToolAsync( - ISandbox sandbox, - string workingDirectory, - string tool, - ExternalToolAuditorOptions options, - CancellationToken ct) - { - await ThrowIfToolVersionMismatchAsync(sandbox, workingDirectory, tool, options, ct) - .ConfigureAwait(false); - } - - private async Task ThrowIfToolVersionMismatchAsync( - ISandbox sandbox, - string workingDirectory, - string tool, - ExternalToolAuditorOptions options, - CancellationToken ct) - { - var configured = _expectedVersion(); - var expected = NormalizeVersion(configured); - if (expected is null) - throw new AuditUnavailableException( - $"could-not-verify: auditor '{Name}' has an unparseable ExpectedVersion " - + $"('{TruncateForMessage(configured)}'); set CodeyBox:Plugins:{PluginId}:ExpectedVersion " - + $"to an {tool} release such as '{DefaultExpectedVersion}'.") - { IsDeterministic = true }; - - var result = await ExecToolBoundedAsync( - sandbox, - tool, - "version check", - new SandboxExec - { - Argv = [tool, "--version"], - WorkingDirectory = workingDirectory, - MaxStdoutBytes = ProbeMaxOutputBytes, - MaxStderrBytes = ProbeMaxOutputBytes, - KillOnOutputLimit = true, - }, - ProbeTimeout(options), - ct).ConfigureAwait(false); - - var reported = ExtractVersion(result.Stdout); - if (result.ExecutionUnavailable - || result.ExitCode != 0 - || reported is null) - throw new AuditUnavailableException( - $"could-not-verify: audit tool '{tool}' version could not be determined " - + $"(exit {result.ExitCode}). The pinned release is required before the scan can run — " - + $"a missing or foreign '{tool}' is infrastructure, not a verdict on the diff.", - result.ExitCode, - result.Stdout + "\n" + result.Stderr); - - if (!string.Equals(reported, expected, StringComparison.Ordinal)) - throw new AuditUnavailableException( - $"could-not-verify: audit tool '{tool}' is version {reported}, but this auditor is " - + $"pinned to {expected}. A different scanner version changes the rule set and the " - + "findings; provision the pinned release or set ExpectedVersion to the version you provisioned.") - { IsDeterministic = true }; - } - - private static TimeSpan ProbeTimeout(ExternalToolAuditorOptions options) - { - var timeout = EffectiveTimeout(options); - return timeout > ProbeTimeoutCap ? ProbeTimeoutCap : timeout; - } - - private static string? ExtractVersion(string stdout) - { - // `eslint --version` prints "v10.10.0" — strip the v prefix. - var match = VersionPattern.Match(stdout); - return match.Success ? match.Value : null; - } - - private static string? NormalizeVersion(string? configured) - { - var value = string.IsNullOrWhiteSpace(configured) - ? DefaultExpectedVersion - : configured.Trim(); - var match = VersionPattern.Match(value); - return match.Success ? match.Value : null; - } - - private static string TruncateForMessage(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - return "(empty)"; - var single = SingleLine(value); - return single.Length > MessageValueMaxChars - ? single[..MessageValueMaxChars] + "…" - : single; - } } diff --git a/plugins/auditors-schema/CodeyBox.SpectralAuditorPlugin/SpectralAuditor.cs b/plugins/auditors-schema/CodeyBox.SpectralAuditorPlugin/SpectralAuditor.cs index e206167f3..c85215a39 100644 --- a/plugins/auditors-schema/CodeyBox.SpectralAuditorPlugin/SpectralAuditor.cs +++ b/plugins/auditors-schema/CodeyBox.SpectralAuditorPlugin/SpectralAuditor.cs @@ -1,4 +1,3 @@ -using System.Text.RegularExpressions; using CodeyBox.Core; using CodeyBox.PluginSdk; using CodeyBox.PluginSdk.Tools; @@ -11,8 +10,8 @@ namespace CodeyBox.SpectralAuditorPlugin; /// : the base supplies sandboxed invocation /// with a bounded timeout, per-stream output caps, SARIF parsing, severity mapping, /// exit-code classification, and per-auditor configuration. This class adds Spectral-specific -/// argument assembly and pinned tool version validation through -/// . +/// argument assembly and declares the pinned tool version through +/// . /// /// Gate behaviour: hybrid / severity-driven. Spectral classifies rule /// violations into error, warning, info, and hint. By default, schema violations and @@ -73,12 +72,6 @@ public sealed class SpectralAuditor : ExternalToolAuditorBase, IPluginInitialize public const string TargetPatternsKey = "TargetPatterns"; private const string DefaultGlob = "**/*.{json,yml,yaml}"; - private const int ProbeMaxOutputBytes = 16 * 1024; - private const int MessageValueMaxChars = 64; - private static readonly TimeSpan ProbeTimeoutCap = TimeSpan.FromSeconds(30); - private static readonly Regex VersionPattern = new( - @"\d+\.\d+\.\d+[\w.\-]*", - RegexOptions.CultureInvariant | RegexOptions.Compiled); private static readonly ExternalToolAuditorOptions AuditorDefaults = new() { @@ -123,6 +116,10 @@ public sealed class SpectralAuditor : ExternalToolAuditorBase, IPluginInitialize /// protected override Func OptionsAccessor => _optionsAccessor; + /// + protected override ToolVersionPin? VersionPin => + new(PluginId, _expectedVersion, DefaultExpectedVersion, ["--version"]); + /// protected override IReadOnlyList BuildToolArguments(ExternalToolAuditorOptions options) { @@ -136,8 +133,7 @@ protected override IReadOnlyList BuildToolArguments(ExternalToolAuditorO var ruleset = _rulesetPath(); if (!string.IsNullOrWhiteSpace(ruleset) - && !options.ExtraArguments.Contains("--ruleset", StringComparer.Ordinal) - && !options.ExtraArguments.Contains("-r", StringComparer.Ordinal)) + && !ExtraArgumentsSupplyFlag(options, "--ruleset", "-r")) { args.Add("--ruleset"); args.Add(ruleset.Trim()); @@ -162,114 +158,12 @@ public Task InitializeAsync(PluginContext context, CancellationToken ct = defaul ArgumentNullException.ThrowIfNull(context); var scoped = context.ScopedConfig; _optionsAccessor = () => ExternalToolAuditorOptions.Bind(scoped, AuditorDefaults); - _expectedVersion = () => scoped["ExpectedVersion"]; + _expectedVersion = () => scoped[ToolVersionPin.ExpectedVersionKey]; _rulesetPath = () => scoped[RulesetPathKey] ?? scoped["Ruleset"]; - _targetPatterns = () => SplitList(scoped[TargetPatternsKey] ?? scoped["Documents"]); + _targetPatterns = () => + ExternalToolAuditorOptions.SplitCommaSeparatedList(scoped[TargetPatternsKey] ?? scoped["Documents"]); context.Logger.LogInformation( "SpectralAuditor initialized: pluginId={PluginId}", context.PluginId); return Task.CompletedTask; } - - /// - /// Spectral-specific preconditions on the live path: validates that the installed - /// spectral binary matches the pinned release (ExpectedVersion). - /// - protected override async Task VerifyToolAsync( - ISandbox sandbox, - string workingDirectory, - string tool, - ExternalToolAuditorOptions options, - CancellationToken ct) - { - await ThrowIfToolVersionMismatchAsync(sandbox, workingDirectory, tool, options, ct) - .ConfigureAwait(false); - } - - private async Task ThrowIfToolVersionMismatchAsync( - ISandbox sandbox, - string workingDirectory, - string tool, - ExternalToolAuditorOptions options, - CancellationToken ct) - { - var configured = _expectedVersion(); - var expected = NormalizeVersion(configured); - if (expected is null) - throw new AuditUnavailableException( - $"could-not-verify: auditor '{Name}' has an unparseable ExpectedVersion " - + $"('{TruncateForMessage(configured)}'); set CodeyBox:Plugins:{PluginId}:ExpectedVersion " - + $"to a {tool} release such as '{DefaultExpectedVersion}'.") - { IsDeterministic = true }; - - var result = await ExecToolBoundedAsync( - sandbox, - tool, - "version check", - new SandboxExec - { - Argv = [tool, "--version"], - WorkingDirectory = workingDirectory, - MaxStdoutBytes = ProbeMaxOutputBytes, - MaxStderrBytes = ProbeMaxOutputBytes, - KillOnOutputLimit = true, - }, - ProbeTimeout(options), - ct).ConfigureAwait(false); - - var reported = ExtractVersion(result.Stdout); - if (result.ExecutionUnavailable - || result.ExitCode != 0 - || reported is null) - throw new AuditUnavailableException( - $"could-not-verify: audit tool '{tool}' version could not be determined " - + $"(exit {result.ExitCode}). The pinned release is required before the scan can run — " - + $"a missing or foreign '{tool}' is infrastructure, not a verdict on the diff.", - result.ExitCode, - result.Stdout + "\n" + result.Stderr); - - if (!string.Equals(reported, expected, StringComparison.Ordinal)) - throw new AuditUnavailableException( - $"could-not-verify: audit tool '{tool}' is version {reported}, but this auditor is " - + $"pinned to {expected}. A different scanner version changes the rule set and the " - + "findings; provision the pinned release or set ExpectedVersion to the version you provisioned.") - { IsDeterministic = true }; - } - - private static TimeSpan ProbeTimeout(ExternalToolAuditorOptions options) - { - var timeout = EffectiveTimeout(options); - return timeout > ProbeTimeoutCap ? ProbeTimeoutCap : timeout; - } - - private static string? ExtractVersion(string stdout) - { - var match = VersionPattern.Match(stdout); - return match.Success ? match.Value : null; - } - - private static string? NormalizeVersion(string? configured) - { - var value = string.IsNullOrWhiteSpace(configured) - ? DefaultExpectedVersion - : configured.Trim(); - var match = VersionPattern.Match(value); - return match.Success ? match.Value : null; - } - - private static string TruncateForMessage(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - return "(empty)"; - var single = SingleLine(value); - return single.Length > MessageValueMaxChars - ? single[..MessageValueMaxChars] + "…" - : single; - } - - private static List SplitList(string? value) - => string.IsNullOrWhiteSpace(value) - ? [] - : value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - .Where(static item => item.Length > 0) - .ToList(); } diff --git a/plugins/auditors-secrets/CodeyBox.GitleaksAuditorPlugin/GitleaksAuditor.cs b/plugins/auditors-secrets/CodeyBox.GitleaksAuditorPlugin/GitleaksAuditor.cs index 369d20db9..d0608ec99 100644 --- a/plugins/auditors-secrets/CodeyBox.GitleaksAuditorPlugin/GitleaksAuditor.cs +++ b/plugins/auditors-secrets/CodeyBox.GitleaksAuditorPlugin/GitleaksAuditor.cs @@ -1,5 +1,4 @@ using System.Globalization; -using System.Text.RegularExpressions; using CodeyBox.Core; using CodeyBox.PluginSdk; using CodeyBox.PluginSdk.Tools; @@ -12,9 +11,10 @@ namespace CodeyBox.GitleaksAuditorPlugin; /// : the base supplies sandboxed /// invocation with a bounded timeout, per-stream output caps, SARIF parsing, /// severity mapping, exit-code classification, and per-auditor configuration. -/// This class adds two gitleaks-specific preconditions through the base's -/// seam — the pinned -/// tool version and the repository-suppression gate below. +/// This class declares the pinned tool version through the base's +/// and adds the +/// repository-suppression gate below through +/// . /// /// Gate behaviour: blocking by default. gitleaks has no severity /// vocabulary — every SARIF result is a detected credential — so every finding @@ -120,15 +120,6 @@ public sealed class GitleaksAuditor : ExternalToolAuditorBase, IPluginInitialize private const string ConfigEnvVar = "GITLEAKS_CONFIG_TOML"; private const string PinnedDefaultConfigToml = "[extend]\nuseDefault = true\n"; - private const int ProbeMaxOutputBytes = 16 * 1024; - private const int MessageValueMaxChars = 64; - // The precondition probes are liveness checks, not the scan: they never - // need more than this and share the operator-configured timeout below it. - private static readonly TimeSpan ProbeTimeoutCap = TimeSpan.FromSeconds(30); - private static readonly Regex VersionPattern = new( - @"\d+\.\d+\.\d+[\w.\-]*", - RegexOptions.CultureInvariant | RegexOptions.Compiled); - private static readonly ExternalToolAuditorOptions AuditorDefaults = new() { FindingsExitCodes = new HashSet { 0, LeaksFoundExitCode }, @@ -165,6 +156,10 @@ public sealed class GitleaksAuditor : ExternalToolAuditorBase, IPluginInitialize /// protected override Func OptionsAccessor => _optionsAccessor; + /// + protected override ToolVersionPin? VersionPin => + new(PluginId, _expectedVersion, DefaultExpectedVersion, ["version"]); + /// protected override IReadOnlyList BuildToolArguments(ExternalToolAuditorOptions options) { @@ -214,7 +209,7 @@ public Task InitializeAsync(PluginContext context, CancellationToken ct = defaul ArgumentNullException.ThrowIfNull(context); var scoped = context.ScopedConfig; _optionsAccessor = () => ExternalToolAuditorOptions.Bind(scoped, AuditorDefaults); - _expectedVersion = () => scoped["ExpectedVersion"]; + _expectedVersion = () => scoped[ToolVersionPin.ExpectedVersionKey]; _trustRepositorySuppression = () => bool.TryParse(scoped[TrustRepositorySuppressionKey], out var trust) && trust; context.Logger.LogInformation( @@ -223,11 +218,14 @@ public Task InitializeAsync(PluginContext context, CancellationToken ct = defaul } /// - /// gitleaks-specific preconditions on the live path: the pinned scanner - /// version, and — unless the operator opted in — absence of the + /// gitleaks-specific preconditions on the live path beyond the base's + /// pinned version check: unless the operator opted in, absence of the /// repository-controlled files gitleaks would honor or exempt from the - /// scan (.gitleaksignore, .gitleaks.toml). Both fail closed - /// as infrastructure before the scan runs. + /// scan (.gitleaksignore, .gitleaks.toml) is confirmed + /// through the shared fail-closed presence probe — and + /// .gitleaks.toml is checked in git history too, since the + /// config-path exemption covers every commit. Both fail closed as + /// infrastructure before the scan runs. /// protected override async Task VerifyToolAsync( ISandbox sandbox, @@ -236,63 +234,11 @@ protected override async Task VerifyToolAsync( ExternalToolAuditorOptions options, CancellationToken ct) { - await ThrowIfToolVersionMismatchAsync(sandbox, workingDirectory, tool, options, ct) - .ConfigureAwait(false); if (!_trustRepositorySuppression()) await ThrowIfRepoSuppressionFilePresentAsync(sandbox, workingDirectory, tool, options, ct) .ConfigureAwait(false); } - private async Task ThrowIfToolVersionMismatchAsync( - ISandbox sandbox, - string workingDirectory, - string tool, - ExternalToolAuditorOptions options, - CancellationToken ct) - { - var configured = _expectedVersion(); - var expected = NormalizeVersion(configured); - if (expected is null) - throw new AuditUnavailableException( - $"could-not-verify: auditor '{Name}' has an unparseable ExpectedVersion " - + $"('{TruncateForMessage(configured)}'); set CodeyBox:Plugins:{PluginId}:ExpectedVersion " - + $"to a {tool} release such as '{DefaultExpectedVersion}'.") - { IsDeterministic = true }; - - var result = await ExecToolBoundedAsync( - sandbox, - tool, - "version check", - new SandboxExec - { - Argv = [tool, "version"], - WorkingDirectory = workingDirectory, - MaxStdoutBytes = ProbeMaxOutputBytes, - MaxStderrBytes = ProbeMaxOutputBytes, - KillOnOutputLimit = true, - }, - ProbeTimeout(options), - ct).ConfigureAwait(false); - - var reported = ExtractVersion(result.Stdout); - if (result.ExecutionUnavailable - || result.ExitCode != 0 - || reported is null) - throw new AuditUnavailableException( - $"could-not-verify: audit tool '{tool}' version could not be determined " - + $"(exit {result.ExitCode}). The pinned release is required before the scan can run — " - + $"a missing or foreign '{tool}' is infrastructure, not a verdict on the diff.", - result.ExitCode, - result.Stdout + "\n" + result.Stderr); - - if (!string.Equals(reported, expected, StringComparison.Ordinal)) - throw new AuditUnavailableException( - $"could-not-verify: audit tool '{tool}' is version {reported}, but this auditor is " - + $"pinned to {expected}. A different scanner version changes the rule set and the " - + "findings; provision the pinned release or set ExpectedVersion to the version you provisioned.") - { IsDeterministic = true }; - } - private async Task ThrowIfRepoSuppressionFilePresentAsync( ISandbox sandbox, string workingDirectory, @@ -307,47 +253,23 @@ private async Task ThrowIfRepoSuppressionFilePresentAsync( // would let the subject hide a leak. A repo-root .gitleaks.toml is // gated alongside it: gitleaks exempts its own config path from the // scan, so the file's contents are never checked for secrets at all. - var presence = await ExecToolBoundedAsync( + var present = await ProbeRepositoryFilesPresentAsync( sandbox, + workingDirectory, tool, - "suppression check", - new SandboxExec - { - Argv = - [ - "sh", "-c", - "rc=0; for f in \"$@\"; do if [ -e \"./$f\" ] || [ -L \"./$f\" ]; then echo \"$f\"; rc=1; fi; done; exit $rc", - "sh", RepositoryIgnoreFile, RepositoryConfigFile, - ], - WorkingDirectory = workingDirectory, - MaxStdoutBytes = ProbeMaxOutputBytes, - MaxStderrBytes = ProbeMaxOutputBytes, - KillOnOutputLimit = true, - }, - ProbeTimeout(options), + [RepositoryIgnoreFile, RepositoryConfigFile], + options, ct).ConfigureAwait(false); - - if (presence.ExecutionUnavailable) - throw new AuditUnavailableException( - $"could-not-verify: audit tool '{tool}' suppression check could not run: the sandbox exec " - + "transport was unavailable."); - if (presence.ExitCode != 0) - { - var found = SingleLine(presence.Stdout); - var detail = found.Length > 0 - ? $"found repository-controlled file(s) '{found}'" - : "could not confirm file absence"; + if (present.Count > 0) throw new AuditUnavailableException( - $"could-not-verify: audit tool '{tool}' suppression check {detail} in the audited " - + $"repository (exit {presence.ExitCode}) — gitleaks honors '{RepositoryIgnoreFile}' " - + $"unconditionally and never scans its own '{RepositoryConfigFile}' config path, so " - + "either file lets the audit subject hide a leak. Remove the file(s), or set " + $"could-not-verify: audit tool '{tool}' found repository-controlled file(s) " + + $"'{string.Join("', '", present)}' in the audited repository — gitleaks honors " + + $"'{RepositoryIgnoreFile}' unconditionally and never scans its own " + + $"'{RepositoryConfigFile}' config path, so either file lets the audit subject hide " + + "a leak. Remove the file(s), or set " + $"CodeyBox:Plugins:{PluginId}:{TrustRepositorySuppressionKey} to true to trust " - + "repository-controlled suppression surfaces.", - presence.ExitCode, - presence.Stdout + "\n" + presence.Stderr) + + "repository-controlled suppression surfaces.") { IsDeterministic = true }; - } // The config-path exemption covers every commit, not just the // worktree: a subject could commit a secret inside .gitleaks.toml and @@ -395,35 +317,4 @@ private async Task ThrowIfRepoSuppressionFilePresentAsync( history.Stdout + "\n" + history.Stderr) { IsDeterministic = true }; } - - private static TimeSpan ProbeTimeout(ExternalToolAuditorOptions options) - { - var timeout = EffectiveTimeout(options); - return timeout > ProbeTimeoutCap ? ProbeTimeoutCap : timeout; - } - - private static string? ExtractVersion(string stdout) - { - var match = VersionPattern.Match(stdout); - return match.Success ? match.Value : null; - } - - private static string? NormalizeVersion(string? configured) - { - var value = string.IsNullOrWhiteSpace(configured) - ? DefaultExpectedVersion - : configured.Trim(); - var match = VersionPattern.Match(value); - return match.Success ? match.Value : null; - } - - private static string TruncateForMessage(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - return "(empty)"; - var single = SingleLine(value); - return single.Length > MessageValueMaxChars - ? single[..MessageValueMaxChars] + "…" - : single; - } } diff --git a/src/CodeyBox.PluginSdk/Tools/ExternalToolAuditorBase.cs b/src/CodeyBox.PluginSdk/Tools/ExternalToolAuditorBase.cs index 79add676e..8648cc3d0 100644 --- a/src/CodeyBox.PluginSdk/Tools/ExternalToolAuditorBase.cs +++ b/src/CodeyBox.PluginSdk/Tools/ExternalToolAuditorBase.cs @@ -1,4 +1,5 @@ using System.Text; +using System.Text.RegularExpressions; using CodeyBox.Core; namespace CodeyBox.PluginSdk.Tools; @@ -23,6 +24,26 @@ public abstract class ExternalToolAuditorBase : IAuditor private const int UnavailableOutputTailMaxChars = 4096; private const int TitleMaxChars = 200; private const int MaxBuiltArguments = 256; + private const int MessageValueMaxChars = 64; + + /// Per-stream capture cap for precondition probes (version checks, repository-suppression gates). + protected const int ProbeMaxOutputBytes = 16 * 1024; + + // Precondition probes are liveness checks, not the scan: they never need + // more than this and share the operator-configured timeout below it. + private static readonly TimeSpan ProbeTimeoutCap = TimeSpan.FromSeconds(30); + + private static readonly Regex ToolVersionPattern = new( + @"\d+\.\d+\.\d+[\w.\-]*", + RegexOptions.CultureInvariant | RegexOptions.Compiled); + + // Each candidate is probed with -e (exists) and -L (symlink — catches a + // dangling symlink that -e would miss) and echoed when present; the + // script always exits 0 once it completes, so the exit code carries only + // probe health while stdout carries the verdict. The "./" prefix keeps a + // leading-dash name from being read as a test operator. + private const string RepositoryFilePresenceScript = + "for f in \"$@\"; do if [ -e \"./$f\" ] || [ -L \"./$f\" ]; then printf '%s\\n' \"$f\"; fi; done; exit 0"; /// Stable name for logs and findings. public abstract string Name { get; } @@ -66,15 +87,29 @@ public abstract class ExternalToolAuditorBase : IAuditor protected virtual IReadOnlyDictionary? BuildToolEnvironment(ExternalToolAuditorOptions options) => null; + /// + /// Optional pinned-version declaration. Non-null makes + /// probe the tool with the pin's + /// before every scan — + /// after the presence check, before — and + /// fail closed on a missing binary, an unrecognised version string, or a + /// version other than the configured or default expectation. Default + /// null: no version precondition. + /// + protected virtual ToolVersionPin? VersionPin => null; + /// /// Pre-scan precondition hook, invoked inside after - /// the tool's presence is confirmed and before the scan executes. Override - /// for tool requirements the base cannot express — a pinned version, a - /// repository-state gate — and throw - /// to fail closed: a failed precondition is infrastructure, never a pass. + /// the tool's presence and declared are confirmed + /// and before the scan executes. Override for tool requirements the base + /// cannot express — e.g. a repository-state gate — and throw + /// to fail closed: a failed + /// precondition is infrastructure, never a pass. /// Use for precondition probes so they - /// get the same timeout bounding and failure classification as the scan. - /// The default imposes no extra preconditions. + /// get the same timeout bounding and failure classification as the scan; + /// covers the common + /// "does a repository-controlled file exist" gate. The default imposes no + /// extra preconditions. /// protected virtual Task VerifyToolAsync( ISandbox sandbox, @@ -99,6 +134,7 @@ public async Task RunAsync( var argv = BuildArgv(tool, options); await ThrowIfToolMissingAsync(sandbox, workingDirectory, tool, ct).ConfigureAwait(false); + await VerifyToolVersionPinAsync(sandbox, workingDirectory, tool, options, ct).ConfigureAwait(false); await VerifyToolAsync(sandbox, workingDirectory, tool, options, ct).ConfigureAwait(false); var result = await ExecToolAsync(sandbox, workingDirectory, tool, argv, options, ct).ConfigureAwait(false); @@ -251,6 +287,234 @@ protected static async Task ExecToolBoundedAsync( } } + private async Task VerifyToolVersionPinAsync( + ISandbox sandbox, + string workingDirectory, + string tool, + ExternalToolAuditorOptions options, + CancellationToken ct) + { + if (VersionPin is not { } pin) + return; + + var configured = pin.ConfiguredExpectedVersion(); + var expected = ExtractToolVersion( + string.IsNullOrWhiteSpace(configured) ? pin.DefaultExpectedVersion : configured.Trim()); + if (expected is null) + throw new AuditUnavailableException( + $"could-not-verify: auditor '{Name}' has an unparseable {ToolVersionPin.ExpectedVersionKey} " + + $"('{TruncateForMessage(configured)}'); set CodeyBox:Plugins:{pin.PluginId}:{ToolVersionPin.ExpectedVersionKey} " + + $"to a {tool} release such as '{pin.DefaultExpectedVersion}'.") + { IsDeterministic = true }; + + var probeArguments = pin.VersionProbeArguments.Count > 0 + ? pin.VersionProbeArguments + : ["--version"]; + var result = await ExecToolBoundedAsync( + sandbox, + tool, + "version check", + new SandboxExec + { + Argv = [tool, .. probeArguments], + WorkingDirectory = workingDirectory, + MaxStdoutBytes = ProbeMaxOutputBytes, + MaxStderrBytes = ProbeMaxOutputBytes, + KillOnOutputLimit = true, + }, + ProbeTimeout(options), + ct).ConfigureAwait(false); + + var reported = ExtractToolVersion(result.Stdout); + if (result.ExecutionUnavailable + || result.ExitCode != 0 + || reported is null) + throw new AuditUnavailableException( + $"could-not-verify: audit tool '{tool}' version could not be determined " + + $"(exit {result.ExitCode}). The pinned release is required before the scan can run — " + + $"a missing or foreign '{tool}' is infrastructure, not a verdict on the diff.", + result.ExitCode, + result.Stdout + "\n" + result.Stderr); + + if (!string.Equals(reported, expected, StringComparison.Ordinal)) + throw new AuditUnavailableException( + $"could-not-verify: audit tool '{tool}' is version {reported}, but this auditor is " + + $"pinned to {expected}. A different release changes the tool's checks and its " + + $"findings; provision the pinned release or set {ToolVersionPin.ExpectedVersionKey} " + + "to the version you provisioned.") + { IsDeterministic = true }; + } + + /// + /// Bounded presence probe for repository-controlled files at the audited + /// worktree root — e.g. suppression files the audit subject could use to + /// hide findings from the tool. Returns the subset of + /// that exist (regular files and + /// symlinks). Fails closed: an exec-transport failure or any non-zero + /// probe exit throws — "could not + /// confirm absence" is never treated as "absent". Path entries must be + /// relative; absolute paths, .. segments, and embedded newlines are + /// rejected so the probe can never escape the worktree or corrupt its + /// one-name-per-line protocol. + /// + protected static async Task> ProbeRepositoryFilesPresentAsync( + ISandbox sandbox, + string workingDirectory, + string tool, + IReadOnlyList relativePaths, + ExternalToolAuditorOptions options, + CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(sandbox); + ArgumentNullException.ThrowIfNull(relativePaths); + + var requested = new HashSet(StringComparer.Ordinal); + var argv = new List(relativePaths.Count + 4) + { + "sh", "-c", RepositoryFilePresenceScript, "sh", + }; + foreach (var path in relativePaths) + { + var normalized = NormalizeProbePath(path); + if (requested.Add(normalized)) + argv.Add(normalized); + } + if (requested.Count == 0) + return []; + + var result = await ExecToolBoundedAsync( + sandbox, + tool, + "suppression check", + new SandboxExec + { + Argv = argv, + WorkingDirectory = workingDirectory, + MaxStdoutBytes = ProbeMaxOutputBytes, + MaxStderrBytes = ProbeMaxOutputBytes, + KillOnOutputLimit = true, + }, + ProbeTimeout(options), + ct).ConfigureAwait(false); + + if (result.ExecutionUnavailable) + throw new AuditUnavailableException( + $"could-not-verify: audit tool '{tool}' suppression check could not run: the sandbox exec " + + "transport was unavailable."); + if (result.ExitCode != 0) + throw new AuditUnavailableException( + $"could-not-verify: audit tool '{tool}' suppression check could not confirm repository-file " + + $"absence (exit {result.ExitCode}) — a failed probe is infrastructure, not evidence " + + "that the files are absent.", + result.ExitCode, + result.Stdout + "\n" + result.Stderr); + + // The probe echoes each present path, one per line; intersect with + // the requested set — output beyond it is not trusted. + var present = new List(); + foreach (var line in result.Stdout.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (requested.Contains(line)) + present.Add(line); + } + return present; + } + + /// + /// True when the operator's + /// already supplies + /// any of : the separated "--flag" form, the + /// attached "--flag=value" form, or — for a single-dash short + /// option that takes a value — the joined "-fvalue"/"-f=value" + /// form. Check this before emitting a flag whose repetition would conflict + /// with or silently override the operator's own setting. + /// + protected static bool ExtraArgumentsSupplyFlag( + ExternalToolAuditorOptions options, + params string[] flags) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(flags); + foreach (var flag in flags) + { + if (string.IsNullOrEmpty(flag)) + continue; + var attachedPrefix = flag + "="; + var shortJoined = flag.Length == 2 && flag[0] == '-' && flag[1] != '-'; + foreach (var arg in options.ExtraArguments) + { + if (string.Equals(arg, flag, StringComparison.Ordinal) + || arg.StartsWith(attachedPrefix, StringComparison.Ordinal) + || (shortJoined + && arg.Length > flag.Length + && arg.StartsWith(flag, StringComparison.Ordinal))) + return true; + } + } + return false; + } + + /// + /// Normalizes an + /// entry to its repository-relative form — the single source of truth for + /// the finding-level filter and for auditors translating entries into + /// tool flags. Returns null for blank entries; a preserved trailing + /// / marks a directory-prefix exclusion. + /// + protected static string? NormalizeExcludePathEntry(string? entry) + { + if (string.IsNullOrWhiteSpace(entry)) + return null; + var normalized = entry.Replace('\\', '/').Trim().TrimStart('/'); + return normalized.Length == 0 ? null : normalized; + } + + /// + /// Bounded timeout for precondition probes: shares the configured scan + /// timeout below a fixed cap — a probe is a liveness check, not the scan. + /// + protected static TimeSpan ProbeTimeout(ExternalToolAuditorOptions options) + { + var timeout = EffectiveTimeout(options); + return timeout > ProbeTimeoutCap ? ProbeTimeoutCap : timeout; + } + + /// + /// Extracts the first major.minor.patch version token from tool + /// version output; null when the output carries none. + /// + protected static string? ExtractToolVersion(string output) + { + var match = ToolVersionPattern.Match(output); + return match.Success ? match.Value : null; + } + + private static string NormalizeProbePath(string? path) + { + // Probe paths are baked into the sh script's "$@" list: refuse + // anything that could escape the worktree or break the + // one-name-per-line protocol. + var normalized = (path ?? string.Empty).Replace('\\', '/').Trim(); + if (normalized.Length == 0 + || normalized[0] == '/' + || normalized.IndexOf('\n') >= 0 + || normalized.Split('/').Contains("..", StringComparer.Ordinal)) + throw new ArgumentException( + "Repository probe paths must be relative paths inside the worktree.", + nameof(path)); + return normalized; + } + + private static string TruncateForMessage(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + return "(empty)"; + var single = SingleLine(value); + return single.Length > MessageValueMaxChars + ? single[..MessageValueMaxChars] + "…" + : single; + } + private IReadOnlyList ParseOutput(string tool, SandboxExecResult result) { try @@ -321,10 +585,8 @@ private static bool IsPathExcluded(ExternalToolFinding item, ExternalToolAuditor var path = NormalizeFindingPath(item.Path); foreach (var entry in options.ExcludePaths) { - if (string.IsNullOrWhiteSpace(entry)) - continue; - var normalized = entry.Replace('\\', '/').Trim().TrimStart('/'); - if (normalized.Length == 0) + var normalized = NormalizeExcludePathEntry(entry); + if (normalized is null) continue; if (normalized.EndsWith("/", StringComparison.Ordinal)) { diff --git a/src/CodeyBox.PluginSdk/Tools/ExternalToolAuditorOptions.cs b/src/CodeyBox.PluginSdk/Tools/ExternalToolAuditorOptions.cs index 12179ff94..378279f16 100644 --- a/src/CodeyBox.PluginSdk/Tools/ExternalToolAuditorOptions.cs +++ b/src/CodeyBox.PluginSdk/Tools/ExternalToolAuditorOptions.cs @@ -109,7 +109,7 @@ public static ExternalToolAuditorOptions Bind( if (maxFindings.HasValue && maxFindings.Value > 0) bound.MaxFindings = Math.Min(maxFindings.Value, 100_000); - var exitCodes = SplitList(section["FindingsExitCodes"]) + var exitCodes = SplitCommaSeparatedList(section["FindingsExitCodes"]) .Select(ReadInt) .Where(code => code.HasValue) .Select(code => code!.Value) @@ -126,32 +126,37 @@ public static ExternalToolAuditorOptions Bind( _ => defaults.MinimumSeverity, }; - var included = SplitList(section["IncludedRules"]).ToHashSet(StringComparer.Ordinal); + var included = SplitCommaSeparatedList(section["IncludedRules"]).ToHashSet(StringComparer.Ordinal); if (included.Count > 0) bound.IncludedRules = included; - var excluded = SplitList(section["ExcludedRules"]).ToHashSet(StringComparer.Ordinal); + var excluded = SplitCommaSeparatedList(section["ExcludedRules"]).ToHashSet(StringComparer.Ordinal); if (excluded.Count > 0) bound.ExcludedRules = excluded; - var excludePaths = SplitList(section["ExcludePaths"]); + var excludePaths = SplitCommaSeparatedList(section["ExcludePaths"]); if (excludePaths.Count > 0) bound.ExcludePaths = excludePaths; - var extraArguments = SplitList(section["ExtraArguments"]); + var extraArguments = SplitCommaSeparatedList(section["ExtraArguments"]); if (extraArguments.Count > 0) bound.ExtraArguments = extraArguments.Take(MaxExtraArguments + 1).ToList(); return bound; } - private static int? ReadInt(string? value) - => int.TryParse(value?.Trim(), out var parsed) ? parsed : null; - - private static List SplitList(string? value) + /// + /// Splits a comma-separated scoped-config value into trimmed, non-empty + /// entries — the shared convention for list-valued auditor keys + /// (ExcludePaths, ExtraArguments, plugin-specific lists). + /// + public static List SplitCommaSeparatedList(string? value) => string.IsNullOrWhiteSpace(value) ? [] : value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) .Where(item => item.Length > 0) .ToList(); + + private static int? ReadInt(string? value) + => int.TryParse(value?.Trim(), out var parsed) ? parsed : null; } diff --git a/src/CodeyBox.PluginSdk/Tools/ToolVersionPin.cs b/src/CodeyBox.PluginSdk/Tools/ToolVersionPin.cs new file mode 100644 index 000000000..bff241405 --- /dev/null +++ b/src/CodeyBox.PluginSdk/Tools/ToolVersionPin.cs @@ -0,0 +1,43 @@ +using CodeyBox.Core; + +namespace CodeyBox.PluginSdk.Tools; + +/// +/// Declared version pin for an external-tool auditor: the release the +/// invocation and its report shape were verified against. Declaring a pin via +/// makes +/// probe the tool before every +/// scan and fail closed as when the +/// binary is missing, prints an unrecognised version string, or reports a +/// version other than the configured or default expectation — an unpinned +/// scanner changes its findings under the operator. +/// +/// +/// Plugin id, used in the operator-facing hint that names the scoped-config +/// key (CodeyBox:Plugins:<id>:ExpectedVersion). +/// +/// +/// Resolves the operator-configured ExpectedVersion per invocation so +/// scoped-config edits apply without a restart; may return null or blank to +/// fall back to . +/// +/// +/// Release used when the operator does not configure one. +/// +/// +/// Arguments appended after the tool name for the probe — typically +/// ["--version"], or ["version"] for subcommand-style CLIs. +/// Empty falls back to --version. +/// +public sealed record ToolVersionPin( + string PluginId, + Func ConfiguredExpectedVersion, + string DefaultExpectedVersion, + IReadOnlyList VersionProbeArguments) +{ + /// + /// Scoped-config key every external-tool auditor honors for overriding + /// the pinned tool version. + /// + public const string ExpectedVersionKey = "ExpectedVersion"; +} diff --git a/tests/CodeyBox.Tests/ExternalToolAuditorTests.cs b/tests/CodeyBox.Tests/ExternalToolAuditorTests.cs index e7f362ae8..18bf897c9 100644 --- a/tests/CodeyBox.Tests/ExternalToolAuditorTests.cs +++ b/tests/CodeyBox.Tests/ExternalToolAuditorTests.cs @@ -266,6 +266,68 @@ public void OptionsBind_ReadsOperationalKnobs_WithSafeFallbacks() Assert.Equal(25, bound.MaxFindings); } + [Fact] + public async Task RepositoryFileProbe_ReportsPresent_AndFailsClosedOnProbeError() + { + var auditor = new ProbeExposingAuditor(); + + var execs = 0; + var sandbox = new FakeSandbox((exec, _) => + { + execs++; + return Task.FromResult(new SandboxExecResult(0, ".hiddenrc\n./stray\n", "")); + }); + // Only names that were actually probed count — output beyond the + // requested set is not trusted. + var present = await auditor.ProbeAsync(sandbox, [".hiddenrc", ".other"], CancellationToken.None); + Assert.Equal([".hiddenrc"], present); + Assert.Equal(1, execs); + + // A non-zero probe exit means the check itself failed — never + // "file absent". + var failing = new FakeSandbox((exec, _) => + Task.FromResult(new SandboxExecResult(2, "", "sh: syntax error"))); + await Assert.ThrowsAsync( + () => auditor.ProbeAsync(failing, [".hiddenrc"], CancellationToken.None)); + + var unavailable = new FakeSandbox((exec, _) => + Task.FromResult(new SandboxExecResult(0, "", "", ExecutionUnavailable: true))); + await Assert.ThrowsAsync( + () => auditor.ProbeAsync(unavailable, [".hiddenrc"], CancellationToken.None)); + } + + [Fact] + public async Task RepositoryFileProbe_RejectsPathsOutsideTheWorktree() + { + var auditor = new ProbeExposingAuditor(); + var noop = new FakeSandbox((exec, _) => + Task.FromResult(new SandboxExecResult(0, "", ""))); + + await Assert.ThrowsAsync( + () => auditor.ProbeAsync(noop, ["../escape"], CancellationToken.None)); + await Assert.ThrowsAsync( + () => auditor.ProbeAsync(noop, ["/etc/passwd"], CancellationToken.None)); + await Assert.ThrowsAsync( + () => auditor.ProbeAsync(noop, ["a\nb"], CancellationToken.None)); + Assert.Empty(await auditor.ProbeAsync(noop, ["ok.md"], CancellationToken.None)); + } + + [Fact] + public void ExtraArgumentsSupplyFlag_MatchesSeparatedAttachedAndJoinedForms() + { + static ExternalToolAuditorOptions With(string arg) => new() { ExtraArguments = [arg] }; + + Assert.True(ProbeExposingAuditor.SuppliesFlag(With("--config"), "--config")); + Assert.True(ProbeExposingAuditor.SuppliesFlag(With("--config=/x.toml"), "--config")); + Assert.True(ProbeExposingAuditor.SuppliesFlag(With("-c/x.toml"), "-c")); + Assert.True(ProbeExposingAuditor.SuppliesFlag(With("-c=/x.toml"), "-c")); + Assert.True(ProbeExposingAuditor.SuppliesFlag(With("-c/x.toml"), "--config", "-c")); + Assert.False(ProbeExposingAuditor.SuppliesFlag(With("--configurations"), "--config")); + Assert.False(ProbeExposingAuditor.SuppliesFlag(With("--config"), "-c")); + Assert.False(ProbeExposingAuditor.SuppliesFlag(With("--other"), "--config", "-c")); + Assert.False(ProbeExposingAuditor.SuppliesFlag(new ExternalToolAuditorOptions(), "--config")); + } + [Fact] public void InvalidToolName_IsRejectedFailClosed() { @@ -319,6 +381,22 @@ private sealed class TestToolAuditor( protected override IReadOnlyList BuildToolArguments(ExternalToolAuditorOptions toolOptions) => ["scan", "."]; } + private sealed class ProbeExposingAuditor : ExternalToolAuditorBase + { + public override string Name => "test:probe"; + protected override string ToolName => "fictional-scanner"; + protected override IExternalToolOutputParser OutputParser { get; } = new SarifToolOutputParser(); + protected override IReadOnlyList BuildToolArguments(ExternalToolAuditorOptions options) => ["scan", "."]; + + public Task> ProbeAsync( + ISandbox sandbox, IReadOnlyList paths, CancellationToken ct) + => ProbeRepositoryFilesPresentAsync( + sandbox, "/work", ToolName, paths, new ExternalToolAuditorOptions(), ct); + + public static bool SuppliesFlag(ExternalToolAuditorOptions options, params string[] flags) + => ExtraArgumentsSupplyFlag(options, flags); + } + private sealed class FakeSandbox(Func> onExec) : ISandbox { private readonly Func> _onExec = onExec; diff --git a/tests/CodeyBox.Tests/GitleaksAuditorTests.cs b/tests/CodeyBox.Tests/GitleaksAuditorTests.cs index 77e443cf8..45b5d3847 100644 --- a/tests/CodeyBox.Tests/GitleaksAuditorTests.cs +++ b/tests/CodeyBox.Tests/GitleaksAuditorTests.cs @@ -150,7 +150,7 @@ public async Task RepoGitleaksIgnore_FailsClosed_ScanNeverRuns() if (IsPresenceProbe(exec) || IsVersionProbe(exec)) return Task.FromResult(Ok(exec)); if (IsWorktreeSuppressionProbe(exec)) - return Task.FromResult(new SandboxExecResult(1, ".gitleaksignore\n", "")); // file present + return Task.FromResult(new SandboxExecResult(0, ".gitleaksignore\n", "")); // file present if (IsHistorySuppressionProbe(exec)) return Task.FromResult(new SandboxExecResult(0, "", "")); scanExecs++; @@ -178,7 +178,7 @@ public async Task RepoGitleaksToml_FailsClosed_ScanNeverRuns() if (IsPresenceProbe(exec) || IsVersionProbe(exec)) return Task.FromResult(Ok(exec)); if (IsWorktreeSuppressionProbe(exec)) - return Task.FromResult(new SandboxExecResult(1, ".gitleaks.toml\n", "")); // file present + return Task.FromResult(new SandboxExecResult(0, ".gitleaks.toml\n", "")); // file present if (IsHistorySuppressionProbe(exec)) return Task.FromResult(new SandboxExecResult(0, "", "")); scanExecs++; @@ -241,7 +241,7 @@ await auditor.InitializeAsync( if (IsSuppressionProbe(exec)) { suppressionProbes++; - return Task.FromResult(new SandboxExecResult(1, "", "")); // file present + return Task.FromResult(new SandboxExecResult(0, ".gitleaksignore\n", "")); // file present } if (IsPresenceProbe(exec) || IsVersionProbe(exec)) return Task.FromResult(Ok(exec)); diff --git a/tests/CodeyBox.Tests/LycheeAuditorTests.cs b/tests/CodeyBox.Tests/LycheeAuditorTests.cs index b43edb5d2..738fe09d2 100644 --- a/tests/CodeyBox.Tests/LycheeAuditorTests.cs +++ b/tests/CodeyBox.Tests/LycheeAuditorTests.cs @@ -274,6 +274,9 @@ public async Task Fixture_WithBrokenLinks_YieldsFindings_WithRuleIdAndLocation() Assert.Contains("--include-fragments", scanExec.Argv); // Offline by default — no network in the audit sandbox. Assert.Contains("--offline", scanExec.Argv); + // Walker ignore files (.gitignore/.ignore) cannot hide committed + // documentation from the crawl. + Assert.Contains("--no-ignore", scanExec.Argv); // Repo-authored lychee config is pinned out; /dev/null parses empty. Assert.Contains("--config", scanExec.Argv); Assert.Contains("/dev/null", scanExec.Argv); @@ -421,8 +424,9 @@ public async Task LycheeignorePresent_IsInfrastructureFailure_FailsClosed() if (IsPresenceProbe(exec) || IsVersionProbe(exec)) return Task.FromResult(Ok(exec)); if (IsIgnoreProbe(exec)) - // exit 0 = .lycheeignore exists at the repo root - return Task.FromResult(new SandboxExecResult(0, "", "")); + // The probe echoes each present path: stdout carries the + // verdict, exit 0 means the probe itself completed. + return Task.FromResult(new SandboxExecResult(0, ".lycheeignore\n", "")); scanExecs++; return Task.FromResult(new SandboxExecResult(0, JsonClean, "")); }); @@ -435,6 +439,31 @@ public async Task LycheeignorePresent_IsInfrastructureFailure_FailsClosed() Assert.Equal(0, scanExecs); } + [Fact] + public async Task IgnoreProbeError_IsInfrastructureFailure_FailsClosed() + { + var scanExecs = 0; + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec)) + return Task.FromResult(Ok(exec)); + if (IsIgnoreProbe(exec)) + // A non-zero probe exit means the check itself failed — + // "could not confirm absence" is never treated as "absent". + return Task.FromResult(new SandboxExecResult(2, "", "sh: syntax error")); + scanExecs++; + return Task.FromResult(new SandboxExecResult(0, JsonClean, "")); + }); + + IAuditor auditor = new LycheeAuditor(); + var ex = await Assert.ThrowsAsync( + () => auditor.RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None)); + + Assert.Contains("lychee", ex.Message, StringComparison.Ordinal); + Assert.Contains("could not confirm", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, scanExecs); + } + [Fact] public async Task TrustRepositorySuppression_SkipsIgnoreGate_AndDropsConfigPin() { @@ -467,6 +496,7 @@ await auditor.InitializeAsync( Assert.Equal(0, ignoreProbes); Assert.NotNull(scanExec); Assert.DoesNotContain("--config", scanExec!.Argv); + Assert.DoesNotContain("--no-ignore", scanExec.Argv); } [Fact] @@ -499,6 +529,39 @@ await auditor.InitializeAsync( Assert.DoesNotContain("/dev/null", argv); } + [Theory] + [InlineData("--config=/opt/operator.toml")] + [InlineData("-c/opt/operator.toml")] + public async Task ExtraArguments_AttachedConfigForm_SuppressesPin(string extraArg) + { + // An operator --config in the attached/joined form must be detected + // just like the separated form — otherwise the auditor would emit a + // duplicate flag and the documented precedence would not hold. + SandboxExec? scanExec = null; + var sandbox = new FakeSandbox((exec, _) => + { + if (IsPresenceProbe(exec) || IsVersionProbe(exec) || IsIgnoreProbe(exec)) + return Task.FromResult(Ok(exec)); + scanExec = exec; + return Task.FromResult(new SandboxExecResult(0, JsonClean, "")); + }); + + var auditor = new LycheeAuditor(); + await auditor.InitializeAsync( + BuildPluginContext(new Dictionary + { + ["Scoped:ExtraArguments"] = extraArg, + }), + CancellationToken.None); + + await ((IAuditor)auditor).RunAsync(sandbox, "/work", FakeContext(), CancellationToken.None); + + Assert.NotNull(scanExec); + Assert.Contains(extraArg, scanExec!.Argv); + Assert.DoesNotContain("--config", scanExec.Argv); + Assert.DoesNotContain("/dev/null", scanExec.Argv); + } + [Fact] public async Task ScopedConfiguration_Inputs_ReplaceDefaultWholeTreeScope() { @@ -628,11 +691,6 @@ public void DisabledPlugin_IsNotLoaded_AndToolAbsentFromBaselineProvisioning() var tools = loader.GetEnabledPluginTools(); Assert.Empty(tools); - - var contributions = PluginBaselineProvisioning.BuildContributions(tools); - var flattened = string.Join("\n", contributions.InstallCommands) - + "\n" + string.Join("\n", contributions.VerificationCommands.SelectMany(static v => v.Argv)); - Assert.DoesNotContain("lychee", flattened, StringComparison.Ordinal); } [Fact] @@ -771,10 +829,8 @@ private static PluginContext BuildPluginContext(IReadOnlyDictionary IsVersionProbe(exec) ? new SandboxExecResult(0, "lychee " + LycheeAuditor.DefaultExpectedVersion + "\n", "") - : IsIgnoreProbe(exec) - // exit 1 = .lycheeignore absent - ? new SandboxExecResult(1, "", "") - : new SandboxExecResult(0, "", ""); + // Presence probes: exit 0 with empty stdout = probed files absent. + : new SandboxExecResult(0, "", ""); private static bool IsPresenceProbe(SandboxExec exec) => exec.Argv.Count >= 3 @@ -826,12 +882,18 @@ await File.WriteAllTextAsync( }; psi.ArgumentList.Add("--version"); using var process = Process.Start(psi)!; - var stdout = process.StandardOutput.ReadToEnd(); + // Drain both streams concurrently: a full stderr pipe would block + // the child on write while stdout stays open, deadlocking the + // synchronous read ahead of the timeout. + var stdoutTask = process.StandardOutput.ReadToEndAsync(); + var stderrTask = process.StandardError.ReadToEndAsync(); if (!process.WaitForExit(milliseconds: 10_000)) { try { process.Kill(); } catch { /* best-effort probe teardown */ } return null; } + Task.WhenAll(stdoutTask, stderrTask).Wait(TimeSpan.FromSeconds(5)); + var stdout = stdoutTask.IsCompletedSuccessfully ? stdoutTask.Result : string.Empty; var match = Regex.Match(stdout, @"\d+\.\d+\.\d+[\w.\-]*"); return process.ExitCode == 0 && match.Success ? match.Value : null; }