diff --git a/CHANGELOG.md b/CHANGELOG.md index a7a280a1320..8f574aee014 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -108,6 +108,15 @@ Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-l - Fix `LINTER_RULES_PATH` not being used to resolve config files for linters using `active_only_if_file_found` (e.g. `REPOSITORY_LS_LINT`, `SPELL_PROSELINT`, `SPELL_VALE`), fixes [#8416](https://github.com/oxsecurity/megalinter/issues/8416) - Honor `EXCLUDED_DIRECTORIES` and `ADDITIONAL_EXCLUDED_DIRECTORIES` in changed-files mode (`VALIDATE_ALL_CODEBASE: false`), so files inside excluded directories are pruned from the `git diff` file list the same way they are during full-codebase validation ([#8360](https://github.com/oxsecurity/megalinter/issues/8360)) - Declare `COPYPASTE_JSCPD` as `linux/amd64` only: since jscpd v5 the tool is a Rust binary shipped through per-platform npm packages, and upstream publishes no `linux-arm64-musl` target, so it exits with `Unsupported platform linux/arm64` on the Alpine-based ARM images + - Fix sporadic `ENOENT` crashes when `COPYPASTE_JSCPD` and `REPOSITORY_SECRETLINT` run in parallel: jscpd now writes its report to a temporary folder outside the linted workspace, and MegaLinter no longer deletes report files while other linters are still scanning them, fixes [#3979](https://github.com/oxsecurity/megalinter/issues/3979). As jscpd no longer removes its report at the end of a clean run, a copy-paste report left by a previous run is now removed when jscpd starts, so the report folder never mixes results from two runs + - Fix `REPOSITORY_SECRETLINT` silently narrowing its scan when `REPOSITORY_SECRETLINT_ARGUMENTS` names a `--secretlintignore` file that does not exist: MegaLinter no longer falls back to `.secretlintignore` or `.gitignore` in its place, as those patterns often exclude the very files a secrets scanner must inspect + - Fix the `copy-paste` report folder being created unreadable to non-root users (such as a later artifact-upload step), as it inherited the private permissions of the temporary folder jscpd writes to + - Stop `REPOSITORY_SECRETLINT` from scanning MegaLinter's own report folder, which could raise false positives on secrets echoed into other linters' reports, using an ignore file generated inside the report folder so nothing is ever written to or deleted from the linted sources + - Always exclude `REPORT_OUTPUT_FOLDER` from linted directories, even when `EXCLUDED_DIRECTORIES` is overridden + - Report a linter that crashes before producing SARIF output as a failure showing its raw output, instead of a spurious single finding accompanied by a SARIF parsing error + - Stop reporting a linter that exits without parsable SARIF output as a clean success: its results could not be counted, so it is now reported as a warning rather than as zero findings. Applies to linters declaring `can_output_sarif` when SARIF output is enabled, including security scanners such as `REPOSITORY_SECRETLINT`, `REPOSITORY_SEMGREP` and `REPOSITORY_TRIVY` + - Fix `REPOSITORY_SECRETLINT` ignoring a `.secretlintignore` located in `LINTER_RULES_PATH` (e.g. the default `.github/linters`): only its base name was passed, so secretlint resolved it from the workspace root and applied either no patterns at all or those of a different file with the same name + - Keep applying `REPOSITORY_SECRETLINT` ignore patterns when `REPORT_OUTPUT_FOLDER` is disabled, by generating the merged ignore file in a temporary folder outside the linted sources instead of giving up on it - Reporters diff --git a/README.md b/README.md index 553a3392c15..a660e4a698a 100644 --- a/README.md +++ b/README.md @@ -1285,7 +1285,7 @@ description: List of common variables that you can use to customize MegaLinter b | [**ENABLE_ERRORS_LINTERS**](https://github.com/oxsecurity/megalinter/tree/main/docs/config-activation.md) | | List of enabled and blocking linters keys [(more info)](https://github.com/oxsecurity/megalinter/tree/main/docs/config-activation.md) | | [**ENABLE**](https://github.com/oxsecurity/megalinter/tree/main/docs/config-activation.md) | | List of enabled descriptors keys [(more info)](https://github.com/oxsecurity/megalinter/tree/main/docs/config-activation.md) | | [**ENABLE_LINTERS**](https://github.com/oxsecurity/megalinter/tree/main/docs/config-activation.md) | | List of enabled linters keys [(more info)](https://github.com/oxsecurity/megalinter/tree/main/docs/config-activation.md) | -| **EXCLUDED_DIRECTORIES** | \[…many values…\] | List of excluded directory basenames. They're excluded at any nested level. | +| **EXCLUDED_DIRECTORIES** | \[…many values…\] | List of excluded directory basenames. They're excluded at any nested level. The value of **REPORT_OUTPUT_FOLDER** is always excluded in addition to this list. | | **EXTENDS** | | Base `mega-linter.yml` config file(s) to extend local configuration from. Can be a single URL or a list of `.mega-linter.yml` config files URLs. Later files take precedence. | | **FAIL_IF_MISSING_LINTER_IN_FLAVOR** | `false` | If set to `true`, MegaLinter fails if a linter is missing in the selected flavor | | **FAIL_IF_UPDATED_SOURCES** | `false` | If set to `true`, MegaLinter fails if a linter or formatter has autofixed sources, even if there are no errors | @@ -1415,6 +1415,8 @@ ADDITIONAL_EXCLUDED_DIRECTORIES: - vendor ``` +MegaLinter's own report folder (**REPORT_OUTPUT_FOLDER**, `megalinter-reports` by default) is always excluded from linting, even if you override **EXCLUDED_DIRECTORIES**. Linting MegaLinter's own output produces false positives, and the reporters write to that folder while linters are still running. + diff --git a/mega-linter-runner/lib/megalinter-vars.json b/mega-linter-runner/lib/megalinter-vars.json index e45530de027..62d3bec1037 100644 --- a/mega-linter-runner/lib/megalinter-vars.json +++ b/mega-linter-runner/lib/megalinter-vars.json @@ -7849,7 +7849,7 @@ "name": "EXCLUDED_DIRECTORIES", "type": "array", "title": "Excluded directories", - "description": "List of excluded directory basenames replacing MegaLinter default excluded directories. They are excluded at any nested level.", + "description": "List of excluded directory basenames replacing MegaLinter default excluded directories. They are excluded at any nested level. The value of REPORT_OUTPUT_FOLDER is always excluded in addition to this list.", "examples": [ [ ".github", diff --git a/megalinter/Linter.py b/megalinter/Linter.py index c0285d20a76..b5c37230c48 100644 --- a/megalinter/Linter.py +++ b/megalinter/Linter.py @@ -63,6 +63,9 @@ def __init__(self, params=None, linter_config=None): self.linter_speed = 3 self.can_output_sarif = False self.output_sarif = False + # Sticky flag: set when SARIF output could not be parsed, so a linter that + # exits 0 with unparsable SARIF is not reported as clean (see get_sarif_result_count) + self.sarif_parse_failed = False # ex: https://eslint.org/ self.linter_url = ( "Field 'linter_url' must be overridden at custom linter class level" @@ -940,6 +943,18 @@ def run( if self.cli_lint_mode == "list_of_files": self.update_files_lint_results(self.files, None, None, None, None, None) + # A linter that exited without parsable SARIF has not been measured, so + # counting zero results would report it as clean. Every reporter still shows + # it with zero errors and zero warnings, hence the log: the status alone + # cannot tell an operator that the results are missing rather than empty. + if self.status == "success" and self.sarif_parse_failed is True: + self.status = "warning" + logging.warning( + f"[{self.linter_name}] results could not be counted, as its SARIF " + "output was not parsable: reporting this linter as a warning rather " + "than as a clean success" + ) + # Set return code to 0 if failures in this linter must not make the MegaLinter run fail if self.return_code != 0: # Disable errors: no failure, just warning @@ -1693,8 +1708,20 @@ def get_sarif_result_count(self, stdout: str, level: str): sarif_output = yaml.safe_load(sarif_file) # SARIF is in stdout else: - # SARIF is in stdout - sarif_output = yaml.safe_load(stdout) + # SARIF is in stdout. Require it to actually have SARIF shape: + # find_json_in_stdout returns "" unless the payload parses as JSON + # containing a "runs" key. A linter that crashed before producing + # SARIF leaves a stack trace here, which must surface as a failure + # rather than be parsed as results + sarif_stdout = utils.find_json_in_stdout(stdout) + if sarif_stdout == "": + logging.error( + f"[{self.linter_name}] exited without producing parsable SARIF " + f"output while counting {level}s.\nLinter output: {stdout}" + ) + self.sarif_parse_failed = True + return 0 + sarif_output = json.loads(sarif_stdout) for run in sarif_output["runs"]: rule_default_level_map = {} @@ -1737,14 +1764,15 @@ def get_sarif_result_count(self, stdout: str, level: str): return total_result except Exception as e: - total_result = 1 + # Return 0 rather than inventing a finding: get_total_number_errors + # already reports 1 error for any non-success linter, so a broken linter + # still fails the run without being attributed a phantom result logging.error( - f"Error while getting total {level}s from SARIF output.\nError:" - + str(e) - + "\nstdout: " - + stdout + f"[{self.linter_name}] unable to compute total {level}s from SARIF " + f"output.\nError: {str(e)}\nLinter output: {stdout}" ) - return total_result + self.sarif_parse_failed = True + return 0 # Build the CLI command to get linter version (can be overridden if --version is not the way to get the version) def build_version_command(self): diff --git a/megalinter/descriptors/repository.megalinter-descriptor.yml b/megalinter/descriptors/repository.megalinter-descriptor.yml index 6e84f33adff..99eb29a58dd 100644 --- a/megalinter/descriptors/repository.megalinter-descriptor.yml +++ b/megalinter/descriptors/repository.megalinter-descriptor.yml @@ -633,6 +633,15 @@ linters: - **Team Collaboration**: Shared configuration files ensure consistent secret detection across development teams - **Performance Optimized**: Fast scanning suitable for large repositories and automated workflows - **Open Source**: Community-driven with transparent detection rules and continuous updates + + **Ignore patterns in MegaLinter:** + + MegaLinter does not pass your ignore file to secretlint directly. It generates a `.secretlintignore-megalinter` file inside **REPORT_OUTPUT_FOLDER** and passes that instead, because secretlint applies only one ignore file. + + - The generated file merges the patterns of your own ignore file: the one named by `REPOSITORY_SECRETLINT_ARGUMENTS: --secretlintignore ` if you set it, otherwise your `.secretlintignore`, otherwise your `.gitignore`. Your file is honored wherever it lives, including in **LINTER_RULES_PATH**. + - MegaLinter's own report folder is appended last, so it is always excluded. Ignore rules are last match wins, so this cannot be undone by a negation pattern in your own file. + - Nothing else is added. MegaLinter's **EXCLUDED_DIRECTORIES** are deliberately not merged in: build artifacts such as `.terraform` or `.terragrunt-cache` can contain real credentials baked in at generation time, and a secrets scanner must still see them. + - When **REPORT_OUTPUT_FOLDER** is disabled, the generated file is written to a temporary folder outside your sources instead, so your patterns still apply. linter_url: https://github.com/secretlint/secretlint linter_repo: https://github.com/secretlint/secretlint linter_banner_image_url: https://github.com/secretlint/secretlint/raw/master/docs/assets/SecretLintLP.png @@ -652,12 +661,9 @@ linters: - project cli_lint_extra_args_after: # secretlint v13 introduced a ripgrep-based file walker that respects - # .gitignore by default. --no-gitignore keeps the v12 scan breadth (scan - # everything, filter only via .secretlintignore). The .secretlintignore - # itself (which excludes .automation/test fixtures containing intentional - # fake secrets) is applied via SecretLintLinter.get_ignore_arguments, - # which passes it by base name because the v13 walker matches ignore - # files by base name and drops an absolute path. + # .gitignore by default. --no-gitignore keeps the v12 scan breadth: scan + # everything, filter only via the ignore file MegaLinter generates in + # SecretLintLinter.build_megalinter_ignore_file. # See https://github.com/secretlint/secretlint/releases (v13) - "--no-gitignore" cli_lint_mode_project_extra_args_after: diff --git a/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json b/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json index 15bb3852ec2..0ddef86412f 100644 --- a/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json +++ b/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json @@ -9300,7 +9300,7 @@ }, "EXCLUDED_DIRECTORIES": { "$id": "#/properties/EXCLUDED_DIRECTORIES", - "description": "List of excluded directory basenames replacing MegaLinter default excluded directories. They are excluded at any nested level.", + "description": "List of excluded directory basenames replacing MegaLinter default excluded directories. They are excluded at any nested level. The value of REPORT_OUTPUT_FOLDER is always excluded in addition to this list.", "examples": [ [ ".github", diff --git a/megalinter/linters/JsCpdLinter.py b/megalinter/linters/JsCpdLinter.py index a478a460f93..6dc5b96eb81 100644 --- a/megalinter/linters/JsCpdLinter.py +++ b/megalinter/linters/JsCpdLinter.py @@ -4,19 +4,64 @@ https://github.com/kucherenko/jscpd """ +import logging import os import shutil +import tempfile -from megalinter import Linter, utils +from megalinter import Linter, flavor_factory, utils class JsCpdLinter(Linter): + def __init__(self, params=None, linter_config=None): + self.report_tmp_folder = None + super().__init__(params, linter_config) + self.remove_stale_copy_paste_report() + + # jscpd no longer deletes its report on a clean run, so one left by a previous + # run would outlive the duplication it describes. Construction time is the only + # safe moment to drop it: MegaLinter builds every linter before running any, and + # deleting later would reintroduce the #3979 ENOENT race against linters still + # walking the report folder. is_active does not suffice on its own, as + # check_active_linters_match_flavor only clears it after construction. + def remove_stale_copy_paste_report(self): + if ( + self.is_active is not True + or not self.is_present_in_image_flavor() + or self.report_folder == "" + or not utils.can_write_report_files(self) + ): + return + stale_report_folder = os.path.join(self.report_folder, "copy-paste") + if not os.path.isdir(stale_report_folder): + return + try: + shutil.rmtree(stale_report_folder) + except OSError as e: + logging.warning( + f"[jscpd] Unable to remove the copy-paste report of a previous run " + f"in {stale_report_folder}: {str(e)}" + ) + + # "all" (every linter) and "none" (a single-linter image) have no entry in + # all_flavors.json, so they must return before the lookup, as + # check_active_linters_match_flavor also does. + def is_present_in_image_flavor(self): + flavor = flavor_factory.get_image_flavor() + if flavor in ("all", "none"): + return True + return self.name in flavor_factory.list_flavor_linters(flavor) + # Special cases for build lint command def build_lint_command(self, file=None): if utils.can_write_report_files(self.master): + # jscpd runs while other linters are still scanning the workspace: + # writing the report in place and deleting it again on success used to + # make concurrent project mode linters fail with ENOENT (#3979) + self.report_tmp_folder = tempfile.mkdtemp(prefix="megalinter-jscpd-") self.cli_lint_extra_args += [ "--output", - f"{self.report_folder}/copy-paste/", + self.report_tmp_folder, ] cmd = super().build_lint_command(file) # Do not use Jscpd HTML reporter if deactivated @@ -24,22 +69,52 @@ def build_lint_command(self, file=None): cmd = [item.replace("console,html", "console") for item in cmd] return cmd - # Perform additional actions and provide additional details in text reporter logs - def complete_text_reporter_report(self, reporter_self): - if self.status == "success": - copy_paste_dir = ( - reporter_self.master.report_folder + os.path.sep + "copy-paste" + def process_linter(self, file=None): + # return_code stays 0 if the base call raises, so a crashed run publishes no + # report while its temp folder is still dropped + return_code = 0 + try: + return_code, stdout = super().process_linter(file) + return return_code, stdout + finally: + self.materialize_copy_paste_report(return_code) + + # Nothing is ever removed from the report folder here, so linters still scanning + # the workspace cannot observe a file disappear. + def materialize_copy_paste_report(self, return_code): + if self.report_tmp_folder is None: + return + try: + if return_code != 0 and os.path.isdir(self.report_tmp_folder): + published_folder = os.path.join(self.report_folder, "copy-paste") + shutil.copytree( + self.report_tmp_folder, + published_folder, + dirs_exist_ok=True, + ) + # copytree ends with copystat, copying mkdtemp's 0700 onto the + # published folder whatever the umask and leaving it untraversable + # to a non-root consumer such as a later artifact upload step + os.chmod(published_folder, 0o755) + except OSError as e: + # shutil.Error (partial copy) is an OSError subclass but carries no + # errno/strerror pair, so e.strerror would be None for that case + logging.warning( + f"[jscpd] Unable to copy the copy-paste report into " + f"{self.report_folder}: {str(e)}" ) - if os.path.isdir(copy_paste_dir): - try: - shutil.rmtree(copy_paste_dir) - except OSError as e: - return [ - "", - f"No copy-paste has been detected, but unable to remove {copy_paste_dir}: {e.strerror}", - ] - return [ - "", - "copy-paste folder has been removed, as no excessive copy-paste has been detected", - ] + finally: + shutil.rmtree(self.report_tmp_folder, ignore_errors=True) + self.report_tmp_folder = None + + # Perform additional actions and provide additional details in text reporter logs + def complete_text_reporter_report(self, _reporter_self): + # With report files disabled jscpd runs with the console reporter alone and + # produces nothing whatever it finds, so the absence proves nothing + if self.status == "success" and utils.can_write_report_files(self.master): + return [ + "", + "No excessive copy-paste has been detected, " + "so no copy-paste report has been generated", + ] return [] diff --git a/megalinter/linters/SecretLintLinter.py b/megalinter/linters/SecretLintLinter.py index 4a10c1dca8e..ae5d9515325 100644 --- a/megalinter/linters/SecretLintLinter.py +++ b/megalinter/linters/SecretLintLinter.py @@ -4,22 +4,180 @@ https://github.com/secretlint/secretlint """ +import logging import os +import shutil +import tempfile -from megalinter import Linter, config +from megalinter import Linter, config, utils + +# secretlint's built-in DEFAULT_IGNORE_PATTERNS contains "**/.secretlintignore*", +# so a file with this name is never scanned by secretlint itself +MEGALINTER_IGNORE_FILE_NAME = ".secretlintignore-megalinter" class SecretLintLinter(Linter): - # Called before linting files + def __init__(self, params=None, linter_config=None): + self.megalinter_ignore_file = None + self.ignore_tmp_folder = None + super().__init__(params, linter_config) + + # The only run() override among the linters, because Linter.run() offers no + # post-run hook to extend: it deletes remote_config_file_to_delete and + # remote_ignore_file_to_delete inline. The alternatives do not work either: + # process_linter() is called once per file in file lint mode, so cleaning up + # there would strip the ignore patterns from every file after the first, and + # before_lint_files() runs before the files rather than after. Wrapping run() + # is the only point that runs exactly once, after every file, even when + # process_linter raises. + def run( + self, + run_commands_before_linters=None, + run_commands_after_linters=None, + skip_console_reporter=False, + ): + try: + return super().run( + run_commands_before_linters, + run_commands_after_linters, + skip_console_reporter, + ) + finally: + self.remove_ignore_tmp_folder() + + # Removes the temporary folder created by build_megalinter_ignore_file when + # REPORT_OUTPUT_FOLDER is disabled. + def remove_ignore_tmp_folder(self): + if self.ignore_tmp_folder is None: + return + shutil.rmtree(self.ignore_tmp_folder, ignore_errors=True) + self.ignore_tmp_folder = None + + def before_lint_files(self): + self.megalinter_ignore_file = self.build_megalinter_ignore_file() + + # Generates the single ignore file secretlint will use, merging the user's + # patterns with MegaLinter's report folder. Nothing else is added: narrowing a + # secrets scanner further would hide credentials baked into build artifacts + # (.terraform/modules/modules.json can hold basic-auth module URLs). It lives in + # the report folder, not the workspace, so MegaLinter never creates or removes + # files in the sources being scanned (#3979); with reports disabled it goes to a + # temp folder instead, so a .secretlintignore in LINTER_RULES_PATH still applies. + # secretlint resolves the value as path.join(cwd, value), so it must be relative. + # This whole approach depends on secretlint resolving the patterns *inside* the + # file against the cwd as well, which holds from v13 (the ripgrep-based walker) + # and is why the descriptor pins NPM_SECRETLINT_VERSION to 13.x. Measured on + # 11.3.1, patterns resolved against the ignore file's own directory instead, so + # every line written here -- the user's patterns and the report folder exclusion + # alike -- matched nothing, on a run that still reported success. Re-check this + # before lowering the pin: the failure mode is a secrets scanner quietly + # scanning a different set of files, not an error. + def build_megalinter_ignore_file(self): + reports_disabled = not utils.can_write_report_files(self.master) + if reports_disabled: + self.ignore_tmp_folder = tempfile.mkdtemp(prefix="megalinter-secretlint-") + target_folder = self.ignore_tmp_folder + else: + target_folder = self.report_folder + ignore_file_path = os.path.join(target_folder, MEGALINTER_IGNORE_FILE_NAME) + try: + relative_path = os.path.relpath(ignore_file_path, self.workspace) + report_folder_pattern = ( + None + if reports_disabled + else os.path.relpath(self.report_folder, self.workspace) + ) + except ValueError as e: + logging.warning( + f"[{self.linter_name}] Unable to build a workspace-relative path for " + f"{ignore_file_path} ({str(e)}): MegaLinter reports will not be excluded" + ) + return None + lines = [] + source_ignore_file = self.get_source_ignore_file() + if source_ignore_file is not None: + # Only warn when reports being disabled actually put something at risk: + # a run with no ignore file anywhere never had patterns to preserve + if reports_disabled: + logging.warning( + f"[{self.linter_name}] REPORT_OUTPUT_FOLDER is disabled: the " + f"patterns from {source_ignore_file} are written to a temporary " + "folder instead of the report folder" + ) + # Nothing between here and megalinter/run.py catches exceptions, so an + # unreadable ignore file must degrade rather than kill every other linter + try: + with open(source_ignore_file, "r", encoding="utf-8") as ignore_file: + lines += [ + f"# Copied by MegaLinter from {source_ignore_file}", + ignore_file.read().rstrip("\n"), + ] + except (OSError, UnicodeDecodeError) as e: + logging.warning( + f"[{self.linter_name}] Unable to read {source_ignore_file} " + f"({str(e)}): its ignore patterns will not be applied" + ) + # Appended last on purpose: gitignore semantics are last match wins, so this + # stays authoritative over any user negation pattern. A report folder outside + # the workspace yields a ".." relative path, which is not a valid gitignore + # pattern, and such a folder is never scanned anyway. + if report_folder_pattern is not None and not report_folder_pattern.startswith( + ".." + ): + lines += ["# Added by MegaLinter: never lint MegaLinter's own output"] + lines += [report_folder_pattern.replace(os.path.sep, "/")] + try: + os.makedirs(target_folder, exist_ok=True) + with open(ignore_file_path, "w", encoding="utf-8") as ignore_file: + ignore_file.write("\n".join(lines) + "\n") + except OSError as e: + logging.warning( + f"[{self.linter_name}] Unable to write {ignore_file_path} " + f"({e.strerror}): MegaLinter reports will not be excluded" + ) + return None + return relative_path.replace(os.path.sep, "/") + + # secretlint reads a single ignore file and keeps only the last --secretlintignore + # value, so a user file that is not merged would stop being applied entirely. + def get_source_ignore_file(self): + user_ignore_file = self.get_user_ignore_file_arg() + if user_ignore_file is not None: + if not os.path.isabs(user_ignore_file): + user_ignore_file = os.path.join(self.workspace, user_ignore_file) + if os.path.isfile(user_ignore_file): + return user_ignore_file + # Falling back to .gitignore here would let a typo exclude exactly the + # files a secrets scanner exists to find, with the run still green + logging.warning( + f"[{self.linter_name}] --secretlintignore {user_ignore_file} is not " + "a file: its ignore patterns will not be applied, and no fallback " + "ignore file is used in their place" + ) + return None + if self.ignore_file is not None and os.path.isfile(self.ignore_file): + return self.ignore_file + git_ignore_file = os.path.join(self.workspace, ".gitignore") + if os.path.isfile(git_ignore_file): + return git_ignore_file + return None + + def get_user_ignore_file_arg(self): + for index, arg in enumerate(self.cli_lint_user_args): + if arg == "--secretlintignore" and index + 1 < len(self.cli_lint_user_args): + return self.cli_lint_user_args[index + 1] + if arg.startswith("--secretlintignore="): + return arg.split("=", 1)[1] + return None + def get_ignore_arguments(self, cmd): + if self.megalinter_ignore_file is not None: + return ["--secretlintignore", self.megalinter_ignore_file] + # Only reached when build_megalinter_ignore_file failed to build a relative + # path or to write the file. The absolute path the base class builds never + # resolves, so pass the base name: the walker then finds it at the workspace + # root and applies its patterns from there. ignore_args = super().get_ignore_arguments(cmd) - # secretlint v13+ resolves the --secretlintignore value through its - # ripgrep-style walker (@secretlint/walker), which matches ignore files - # by base name against each scanned directory entry. The absolute path - # MegaLinter builds by default never matches a base name, so the ignore - # file is silently skipped and intentional test fixtures get flagged. - # Pass the base name so the walker discovers the .secretlintignore that - # sits at the workspace root (its patterns then apply from that root). if len(ignore_args) >= 2 and ignore_args[0] == "--secretlintignore": ignore_args = [ "--secretlintignore", @@ -31,14 +189,8 @@ def get_ignore_arguments(self, cmd): if ( len(ignore_args) == 0 and "--secretlintignore" not in self.cli_lint_user_args - and ( - os.path.isfile(os.path.join(self.workspace, ".gitignore")) - and ( - not os.path.isfile( - os.path.join(self.workspace, ".secretlintignore") - ) - ) - ) + and os.path.isfile(os.path.join(self.workspace, ".gitignore")) + and not os.path.isfile(os.path.join(self.workspace, ".secretlintignore")) ): ignore_args = ["--secretlintignore", ".gitignore"] return ignore_args diff --git a/megalinter/tests/test_megalinter/isolated_config_test_case.py b/megalinter/tests/test_megalinter/isolated_config_test_case.py new file mode 100644 index 00000000000..18a00a81238 --- /dev/null +++ b/megalinter/tests/test_megalinter/isolated_config_test_case.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +import os +import unittest +import uuid +from unittest.mock import patch + +from megalinter import config + + +# init_config(request_id, workspace=None) copies os.environ wholesale into the +# config when no workspace is given, so ambient env vars would otherwise leak into +# tests that read configuration (see utils_test.py for the same fix applied inline). +class IsolatedConfigTestCase(unittest.TestCase): + def setUp(self): + self.request_id = str(uuid.uuid1()) + with patch.dict(os.environ, {}, clear=True): + config.init_config(self.request_id) + + def tearDown(self): + config.delete(self.request_id) diff --git a/megalinter/tests/test_megalinter/jscpd_linter_test.py b/megalinter/tests/test_megalinter/jscpd_linter_test.py new file mode 100644 index 00000000000..9efdc85c876 --- /dev/null +++ b/megalinter/tests/test_megalinter/jscpd_linter_test.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python3 +""" +Unit tests for JsCpdLinter report materialization. + +jscpd used to write its report directly into the workspace and MegaLinter deleted +it again on success, which raced with project mode linters still scanning the same +tree (issue #3979). +""" + +import os +import shutil +import stat +import tempfile +import unittest +from unittest.mock import patch + +from megalinter import Linter +from megalinter.linters.JsCpdLinter import JsCpdLinter + + +def _make_linter( + report_folder, report_tmp_folder, is_active=True, name="COPYPASTE_JSCPD" +): + linter = JsCpdLinter.__new__(JsCpdLinter) + linter.report_folder = report_folder + linter.report_tmp_folder = report_tmp_folder + linter.is_active = is_active + linter.name = name + return linter + + +# Reproduce the state left by a jscpd run: a report sitting in the temp folder +# outside the workspace, waiting to be materialized or discarded. jscpd's HTML +# reporter nests its output under an "html" folder (see the real crash stdout +# reproduced in linter_test.py: megalinter-reports/copy-paste/html/jscpd-report.json +# and EmailReporter.py, which filters on "copy-paste/html"), so the fixture must +# reproduce that nesting rather than a flat file. +def _linter_with_generated_report(root): + report_folder = os.path.join(root, "megalinter-reports") + tmp_folder = os.path.join(root, "tmp-jscpd") + os.makedirs(report_folder) + os.makedirs(os.path.join(tmp_folder, "html")) + with open( + os.path.join(tmp_folder, "html", "jscpd-report.html"), "w", encoding="utf-8" + ) as fh: + fh.write("") + # tempfile.mkdtemp always creates its directory 0700 regardless of umask; + # reproduce that here since this fixture builds tmp_folder with os.makedirs + os.chmod(tmp_folder, 0o700) + return _make_linter(report_folder, tmp_folder), report_folder, tmp_folder + + +# Whether a report left behind by a previous run survives +# remove_stale_copy_paste_report when MegaLinter runs under the given image flavor +def _stale_report_survives_flavor(flavor, flavor_linters): + with tempfile.TemporaryDirectory() as root: + report_folder = os.path.join(root, "megalinter-reports") + stale_folder = os.path.join(report_folder, "copy-paste") + os.makedirs(stale_folder) + linter = _make_linter(report_folder, None) + + with ( + patch( + "megalinter.flavor_factory.get_image_flavor", + return_value=flavor, + ), + patch( + "megalinter.flavor_factory.list_flavor_linters", + return_value=flavor_linters, + ), + ): + linter.remove_stale_copy_paste_report() + + return os.path.isdir(stale_folder) + + +class JsCpdLinterTest(unittest.TestCase): + def test_report_materialized_when_clones_found(self): + with tempfile.TemporaryDirectory() as root: + linter, report_folder, tmp_folder = _linter_with_generated_report(root) + + linter.materialize_copy_paste_report(1) + + published_folder = os.path.join(report_folder, "copy-paste") + self.assertTrue( + os.path.isfile( + os.path.join(published_folder, "html", "jscpd-report.html") + ) + ) + self.assertFalse(os.path.isdir(tmp_folder)) + self.assertIsNone(linter.report_tmp_folder) + # shutil.copytree's final copystat(src, dst) would otherwise leave the + # published folder at mkdtemp's 0700, unreadable to a non-root consumer + # such as a later actions/upload-artifact step + self.assertEqual(stat.S_IMODE(os.stat(published_folder).st_mode), 0o755) + + def test_report_discarded_when_no_clones_found(self): + with tempfile.TemporaryDirectory() as root: + linter, report_folder, tmp_folder = _linter_with_generated_report(root) + + linter.materialize_copy_paste_report(0) + + # Nothing is created inside the report folder, and nothing is deleted + # from it either: the copy-paste folder simply never appears + self.assertFalse(os.path.isdir(os.path.join(report_folder, "copy-paste"))) + self.assertFalse(os.path.isdir(tmp_folder)) + self.assertIsNone(linter.report_tmp_folder) + + def test_report_copy_failure_is_logged_and_temp_folder_still_removed(self): + with tempfile.TemporaryDirectory() as root: + linter, report_folder, tmp_folder = _linter_with_generated_report(root) + + with patch.object( + shutil, "copytree", side_effect=PermissionError("denied") + ): + with self.assertLogs(level="WARNING") as logs: + linter.materialize_copy_paste_report(1) + + self.assertTrue(any("Unable to copy" in message for message in logs.output)) + self.assertFalse(os.path.isdir(tmp_folder)) + self.assertIsNone(linter.report_tmp_folder) + + def test_no_temp_folder_is_a_noop(self): + with tempfile.TemporaryDirectory() as root: + report_folder = os.path.join(root, "megalinter-reports") + os.makedirs(report_folder) + + linter = _make_linter(report_folder, None) + linter.materialize_copy_paste_report(1) + + self.assertFalse(os.path.isdir(os.path.join(report_folder, "copy-paste"))) + + def test_stale_report_of_previous_run_is_removed(self): + # CLEAR_REPORT_FOLDER defaults to false, so a report kept from a previous + # run would contradict the text reporter stating that no copy-paste report + # has been generated + with tempfile.TemporaryDirectory() as root: + report_folder = os.path.join(root, "megalinter-reports") + stale_folder = os.path.join(report_folder, "copy-paste") + os.makedirs(stale_folder) + with open( + os.path.join(stale_folder, "jscpd-report.html"), "w", encoding="utf-8" + ) as fh: + fh.write("previous run") + linter = _make_linter(report_folder, None) + + linter.remove_stale_copy_paste_report() + linter.materialize_copy_paste_report(0) + + self.assertFalse(os.path.isdir(stale_folder)) + self.assertTrue(os.path.isdir(report_folder)) + + def test_stale_report_kept_when_linter_is_disabled(self): + # The linter object is still constructed when COPYPASTE_JSCPD is disabled, so + # a disabled jscpd must not discard the report a previous run left behind + with tempfile.TemporaryDirectory() as root: + report_folder = os.path.join(root, "megalinter-reports") + stale_folder = os.path.join(report_folder, "copy-paste") + os.makedirs(stale_folder) + linter = _make_linter(report_folder, None, is_active=False) + + linter.remove_stale_copy_paste_report() + + self.assertTrue(os.path.isdir(stale_folder)) + + def test_stale_report_kept_when_absent_from_image_flavor(self): + # formatters/security don't ship COPYPASTE_JSCPD, but check_active_linters_ + # match_flavor only flips is_active to False after every linter, including + # this one, has already been constructed (MegaLinter.py:227 vs :262) + self.assertTrue(_stale_report_survives_flavor("formatters", ["SPELL_CSPELL"])) + + def test_stale_report_removed_when_present_in_image_flavor(self): + self.assertFalse(_stale_report_survives_flavor("python", ["COPYPASTE_JSCPD"])) + + def test_stale_report_removal_skipped_when_reports_disabled(self): + with tempfile.TemporaryDirectory() as cwd: + os.makedirs(os.path.join(cwd, "copy-paste")) + # A folder that would actually be deleted if report_folder == "none" + # were ever joined with "copy-paste" and resolved against cwd, unlike + # the sibling "copy-paste" folder above which the "" case would delete + os.makedirs(os.path.join(cwd, "none", "copy-paste")) + initial_cwd = os.getcwd() + os.chdir(cwd) + try: + # "none" must never be joined with "copy-paste" and resolved + # against the current directory + _make_linter("none", None).remove_stale_copy_paste_report() + _make_linter("", None).remove_stale_copy_paste_report() + finally: + os.chdir(initial_cwd) + + self.assertTrue(os.path.isdir(os.path.join(cwd, "copy-paste"))) + self.assertTrue(os.path.isdir(os.path.join(cwd, "none", "copy-paste"))) + + def test_clean_run_explains_the_missing_report(self): + linter = _make_linter("megalinter-reports", None) + linter.status = "success" + linter.master = _make_linter("megalinter-reports", None) + + self.assertIn( + "no copy-paste report has been generated", + " ".join(linter.complete_text_reporter_report(None)), + ) + + def test_no_report_explanation_when_report_files_are_disabled(self): + # jscpd then runs with the console reporter alone and writes no report + # whatever it finds, so blaming the absence on a clean run would be wrong + linter = _make_linter("none", None) + linter.status = "success" + linter.master = _make_linter("none", None) + + self.assertEqual(linter.complete_text_reporter_report(None), []) + + def test_no_report_explanation_when_copy_paste_was_found(self): + linter = _make_linter("megalinter-reports", None) + linter.status = "error" + linter.master = _make_linter("megalinter-reports", None) + + self.assertEqual(linter.complete_text_reporter_report(None), []) + + def test_build_lint_command_redirects_output_outside_workspace(self): + # This is the actual #3979 fix: jscpd's report must land outside the + # workspace/report folder other linters are still scanning, not back at + # f"{self.report_folder}/copy-paste/" + with tempfile.TemporaryDirectory() as root: + report_folder = os.path.join(root, "megalinter-reports") + os.makedirs(report_folder) + linter = _make_linter(report_folder, None) + linter.master = linter + linter.cli_lint_extra_args = [] + + with patch.object(Linter, "build_lint_command", return_value=["jscpd"]): + cmd = linter.build_lint_command() + + try: + self.assertIsNotNone(linter.report_tmp_folder) + self.assertTrue(os.path.isdir(linter.report_tmp_folder)) + self.assertFalse(linter.report_tmp_folder.startswith(root + os.sep)) + self.assertFalse( + linter.report_tmp_folder.startswith(report_folder + os.sep) + ) + self.assertIn("--output", linter.cli_lint_extra_args) + output_index = linter.cli_lint_extra_args.index("--output") + self.assertEqual( + linter.cli_lint_extra_args[output_index + 1], + linter.report_tmp_folder, + ) + self.assertEqual(cmd, ["jscpd"]) + finally: + if linter.report_tmp_folder and os.path.isdir(linter.report_tmp_folder): + shutil.rmtree(linter.report_tmp_folder) + + def test_build_lint_command_falls_back_to_console_when_reports_disabled(self): + linter = _make_linter("none", None) + linter.master = linter + linter.cli_lint_extra_args = [] + + with patch.object( + Linter, + "build_lint_command", + return_value=["jscpd", "--reporters", "console,html"], + ): + cmd = linter.build_lint_command() + + self.assertIsNone(linter.report_tmp_folder) + self.assertEqual(linter.cli_lint_extra_args, []) + self.assertEqual(cmd, ["jscpd", "--reporters", "console"]) + + def test_process_linter_materializes_report_on_failure(self): + with tempfile.TemporaryDirectory() as root: + linter, report_folder, tmp_folder = _linter_with_generated_report(root) + + with patch.object( + Linter, "process_linter", return_value=(1, "some stdout") + ): + return_code, stdout = linter.process_linter() + + self.assertEqual((return_code, stdout), (1, "some stdout")) + self.assertTrue( + os.path.isfile( + os.path.join( + report_folder, "copy-paste", "html", "jscpd-report.html" + ) + ) + ) + self.assertFalse(os.path.isdir(tmp_folder)) + self.assertIsNone(linter.report_tmp_folder) + + def test_process_linter_leaks_no_temp_folder_when_base_raises(self): + # Proves the "finally" matters: a plain self.materialize_copy_paste_report() + # placed after super().process_linter() with no try/finally would pass the + # happy-path tests above just as well, but would leak the temp folder created + # by build_lint_command whenever process_linter raises (e.g. Linter.py's + # os.remove of a stale SARIF file before the command is even run) + with tempfile.TemporaryDirectory() as root: + linter, report_folder, tmp_folder = _linter_with_generated_report(root) + + with patch.object( + Linter, "process_linter", side_effect=RuntimeError("boom") + ): + with self.assertRaises(RuntimeError): + linter.process_linter() + + # A crash is not "clones were found": the report must not be published + self.assertFalse(os.path.isdir(os.path.join(report_folder, "copy-paste"))) + self.assertFalse(os.path.isdir(tmp_folder)) + self.assertIsNone(linter.report_tmp_folder) + + def test_process_linter_discards_report_on_success(self): + with tempfile.TemporaryDirectory() as root: + linter, report_folder, tmp_folder = _linter_with_generated_report(root) + + with patch.object(Linter, "process_linter", return_value=(0, "")): + return_code, stdout = linter.process_linter() + + self.assertEqual((return_code, stdout), (0, "")) + self.assertFalse(os.path.isdir(os.path.join(report_folder, "copy-paste"))) + self.assertFalse(os.path.isdir(tmp_folder)) + self.assertIsNone(linter.report_tmp_folder) diff --git a/megalinter/tests/test_megalinter/linter_run_stub.py b/megalinter/tests/test_megalinter/linter_run_stub.py new file mode 100644 index 00000000000..9406883e3c9 --- /dev/null +++ b/megalinter/tests/test_megalinter/linter_run_stub.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +from typing import Any + +from megalinter.Linter import Linter + +# Attributes touched by Linter.run() in project lint mode with SARIF output and +# no pre/post commands/reporters. Shared by linter_test.py and +# secretlint_linter_test.py so the two test files stop hand-maintaining two +# copies of the same knowledge of run()'s attribute surface. +_RUN_STUB_DEFAULTS: dict[str, Any] = { + "linter_name": "test_linter", + "name": "TEST_LINTER", + "unsecured_env_variables": [], + "reporters": [], + "pre_commands": None, + "post_commands": None, + "master": None, + "cli_lint_mode": "project", + "output_sarif": True, + "sarif_output_file": None, + "sarif_default_output_file": None, + "sarif_parse_failed": False, + "cli_lint_errors_count": None, + "cli_lint_warnings_count": None, + "total_number_warnings": 0, + "total_number_errors": 0, + "number_errors": 0, + "status": "success", + "return_code": 0, + "remote_config_file_to_delete": None, + "remote_ignore_file_to_delete": None, + "disable_errors": False, + "disable_errors_if_less_than": None, + "stdout": None, +} + + +def build_project_run_linter(request_id, linter=None, **overrides): + if linter is None: + linter = Linter.__new__(Linter) + if not hasattr(linter, "request_id"): + linter.request_id = request_id + for key, value in _RUN_STUB_DEFAULTS.items(): + if not hasattr(linter, key): + setattr(linter, key, value) + for key, value in overrides.items(): + setattr(linter, key, value) + return linter diff --git a/megalinter/tests/test_megalinter/linter_test.py b/megalinter/tests/test_megalinter/linter_test.py index 78fcb5b8e8a..b3a3343a0b7 100644 --- a/megalinter/tests/test_megalinter/linter_test.py +++ b/megalinter/tests/test_megalinter/linter_test.py @@ -4,15 +4,22 @@ """ -import unittest +import json +import os +import tempfile import uuid from unittest import mock +from unittest.mock import patch from megalinter.Linter import Linter from megalinter.linters.StyleLintLinter import StyleLintLinter +from megalinter.tests.test_megalinter.isolated_config_test_case import ( + IsolatedConfigTestCase, +) +from megalinter.tests.test_megalinter.linter_run_stub import build_project_run_linter -class LinterTest(unittest.TestCase): +class LinterTest(IsolatedConfigTestCase): @staticmethod def build_activation_params(enable_linters, disable_linters, priority): return { @@ -174,19 +181,225 @@ def test_stylelint_does_not_duplicate_user_defined_config_basedir(self): def test_sarif_zero_results_is_not_a_warning(self): linter = Linter.__new__(Linter) + linter.linter_name = "test_linter" linter.sarif_output_file = None linter.sarif_default_output_file = None - sarif = """\ -runs: - - tool: - driver: {} - results: - - level: note - locations: - - physicalLocation: {} -""" + sarif = json.dumps( + { + "runs": [ + { + "tool": {"driver": {}}, + "results": [ + {"level": "note", "locations": [{"physicalLocation": {}}]} + ], + } + ] + } + ) with self.assertNoLogs(level="WARNING"): result = linter.get_sarif_result_count(sarif, "error") self.assertEqual(0, result) + + @staticmethod + def build_sarif_linter(sarif_output_file=None): + linter = Linter.__new__(Linter) + linter.linter_name = "secretlint" + linter.sarif_output_file = sarif_output_file + linter.sarif_default_output_file = None + linter.sarif_parse_failed = False + return linter + + @staticmethod + def build_sarif_payload(): + # One error plus one warning: the fixture both the stdout path and the + # sarif_output_file path count findings against + return json.dumps( + { + "runs": [ + { + "tool": {"driver": {"name": "secretlint"}}, + "results": [ + {"level": "error", "locations": [{}]}, + {"level": "warning", "locations": [{}]}, + ], + } + ] + } + ) + + def test_sarif_count_from_stdout(self): + # Normal path: a linter with findings exits non-zero AND emits valid SARIF + linter = self.build_sarif_linter() + stdout = self.build_sarif_payload() + self.assertEqual(linter.get_sarif_result_count(stdout, "error"), 1) + self.assertEqual(linter.get_sarif_result_count(stdout, "warning"), 1) + + def test_sarif_count_on_crash_output_is_zero(self): + # A linter that died before producing SARIF must not be counted as a finding + linter = self.build_sarif_linter() + stdout = ( + "[Error: ENOENT: no such file or directory, open " + "'/github/workspace/megalinter-reports/copy-paste/html/jscpd-report.json'] {\n" + " errno: -2,\n" + " code: 'ENOENT',\n" + " syscall: 'open',\n" + " path: '/github/workspace/megalinter-reports/copy-paste/html/jscpd-report.json'\n" + "}" + ) + self.assertEqual(linter.get_sarif_result_count(stdout, "error"), 0) + + def test_sarif_count_on_crash_output_logs_raw_output(self): + linter = self.build_sarif_linter() + stdout = "Error: ENOENT: no such file or directory, open 'x.json'" + with self.assertLogs(level="ERROR") as captured: + linter.get_sarif_result_count(stdout, "error") + self.assertIn("secretlint", "\n".join(captured.output)) + self.assertIn("ENOENT", "\n".join(captured.output)) + + def test_sarif_unparsable_stdout_marks_parse_failed(self): + # A linter that exited 0 with a stack trace instead of SARIF must not be + # silently reported as clean: the sticky flag has to be raised + linter = self.build_sarif_linter() + stdout = ( + "TypeError: Cannot read properties of undefined (reading 'map')\n" + " at Object. (/usr/lib/node_modules/jscpd/dist/index.js:1:1)" + ) + result = linter.get_sarif_result_count(stdout, "error") + self.assertEqual(result, 0) + self.assertTrue(linter.sarif_parse_failed) + + def test_sarif_incomplete_structure_marks_parse_failed(self): + # SARIF-shaped output that passes the "runs" gate in find_json_in_stdout but + # is missing "results" must hit the except branch, not be silently swallowed + linter = self.build_sarif_linter() + stdout = json.dumps({"runs": [{"tool": {"driver": {}}}]}) + with self.assertLogs(level="ERROR") as captured: + result = linter.get_sarif_result_count(stdout, "error") + self.assertEqual(result, 0) + self.assertTrue(linter.sarif_parse_failed) + log_text = "\n".join(captured.output) + self.assertIn("secretlint", log_text) + # Distinguishes this from the empty-stdout path: only the except handler + # (triggered by the KeyError on "results") emits this message + self.assertIn("unable to compute total", log_text) + + def test_sarif_valid_output_leaves_parse_failed_false(self): + linter = self.build_sarif_linter() + stdout = json.dumps( + { + "runs": [ + { + "tool": {"driver": {"name": "secretlint"}}, + "results": [ + {"level": "error", "locations": [{}]}, + ], + } + ] + } + ) + linter.get_sarif_result_count(stdout, "error") + self.assertFalse(linter.sarif_parse_failed) + + def test_sarif_count_from_output_file(self): + # The dominant production path: 23 descriptors pass {{SARIF_OUTPUT_FILE}} + # in cli_sarif_args, so the sarif_output_file branch, not stdout, is what + # most linters actually use + sarif = self.build_sarif_payload() + with tempfile.TemporaryDirectory() as tmpdir: + sarif_output_file = os.path.join(tmpdir, "sarif_output.json") + with open(sarif_output_file, "w", encoding="utf-8") as f: + f.write(sarif) + linter = self.build_sarif_linter(sarif_output_file=sarif_output_file) + self.assertEqual(linter.get_sarif_result_count("", "error"), 1) + self.assertEqual(linter.get_sarif_result_count("", "warning"), 1) + self.assertFalse(linter.sarif_parse_failed) + + def test_sarif_empty_output_file_marks_parse_failed(self): + # yaml.safe_load on an empty file returns None, so sarif_output["runs"] + # raises TypeError, which the broad except handler in + # get_sarif_result_count catches + with tempfile.TemporaryDirectory() as tmpdir: + sarif_output_file = os.path.join(tmpdir, "sarif_output.json") + open(sarif_output_file, "w", encoding="utf-8").close() + linter = self.build_sarif_linter(sarif_output_file=sarif_output_file) + with self.assertLogs(level="ERROR") as captured: + result = linter.get_sarif_result_count("", "error") + self.assertEqual(result, 0) + self.assertTrue(linter.sarif_parse_failed) + self.assertIn("secretlint", "\n".join(captured.output)) + + def test_get_total_number_errors_reflects_status_when_unmeasured(self): + # get_total_number_errors falls back to 1 error for any non-success status + # when no count could be extracted from stdout; success stays at 0 + linter = Linter.__new__(Linter) + linter.cli_lint_errors_count = None + linter.output_sarif = False + linter.status = "success" + self.assertEqual(linter.get_total_number_errors("irrelevant stdout"), 0) + + linter.status = "warning" + self.assertEqual(linter.get_total_number_errors("irrelevant stdout"), 1) + + def test_run_project_mode_unparsable_sarif_promotes_to_warning(self): + # End-to-end: a linter that exits 0 while emitting garbage instead of SARIF + # must not be reported as a clean success (issue this change fixes) + linter = build_project_run_linter(self.request_id) + stdout = ( + "TypeError: Cannot read properties of undefined (reading 'map')\n" + " at Object. (/usr/lib/node_modules/jscpd/dist/index.js:1:1)" + ) + with patch.object(Linter, "process_linter", return_value=(0, stdout)): + with self.assertLogs(level="WARNING") as captured: + linter.run() + + self.assertEqual(linter.status, "warning") + self.assertEqual(linter.total_number_errors, 0) + self.assertEqual(linter.total_number_warnings, 0) + self.assertEqual(linter.return_code, 0) + # Every reporter shows this linter with 0 errors and 0 warnings, so the + # promotion has to say why, naming the linter, or a yellow run is unexplained + log_text = "\n".join(captured.output) + self.assertIn("test_linter", log_text) + self.assertIn("results could not be counted", log_text) + + def test_run_project_mode_valid_sarif_zero_results_stays_success(self): + # Negative case proving the promotion is conditional: valid SARIF with no + # findings and exit code 0 must remain "success", not be promoted + linter = build_project_run_linter(self.request_id) + stdout = json.dumps( + { + "runs": [ + { + "tool": {"driver": {}}, + "results": [], + } + ] + } + ) + with patch.object(Linter, "process_linter", return_value=(0, stdout)): + linter.run() + + self.assertEqual(linter.status, "success") + self.assertEqual(linter.total_number_errors, 0) + self.assertEqual(linter.total_number_warnings, 0) + self.assertEqual(linter.return_code, 0) + + def test_run_project_mode_real_failure_not_downgraded_by_unparsable_sarif(self): + # The promotion in Linter.run() must only ever raise a "success" to + # "warning", never override a genuine "error": a linter that both exits + # non-zero AND emits unparsable SARIF has to stay "error", with the + # error count still reflecting the real failure + linter = build_project_run_linter(self.request_id) + stdout = ( + "TypeError: Cannot read properties of undefined (reading 'map')\n" + " at Object. (/usr/lib/node_modules/jscpd/dist/index.js:1:1)" + ) + with patch.object(Linter, "process_linter", return_value=(1, stdout)): + linter.run() + + self.assertEqual(linter.status, "error") + self.assertEqual(linter.total_number_errors, 1) + self.assertEqual(linter.total_number_warnings, 0) + self.assertEqual(linter.return_code, 1) diff --git a/megalinter/tests/test_megalinter/secretlint_linter_test.py b/megalinter/tests/test_megalinter/secretlint_linter_test.py new file mode 100644 index 00000000000..c4fbff12298 --- /dev/null +++ b/megalinter/tests/test_megalinter/secretlint_linter_test.py @@ -0,0 +1,506 @@ +#!/usr/bin/env python3 +""" +Unit tests for the secretlint ignore file MegaLinter generates. + +secretlint resolves --secretlintignore as path.join(cwd, value) and silently skips +the file when that path does not exist, so the value must be relative to the +workspace. The file itself lives in MegaLinter's report folder so MegaLinter never +creates or deletes files inside the sources being scanned (issue #3979). +""" + +import builtins +import errno +import os +import shutil +import tempfile +from unittest.mock import patch + +from megalinter import config +from megalinter.Linter import Linter +from megalinter.linters.SecretLintLinter import ( + MEGALINTER_IGNORE_FILE_NAME, + SecretLintLinter, +) +from megalinter.tests.test_megalinter.isolated_config_test_case import ( + IsolatedConfigTestCase, +) +from megalinter.tests.test_megalinter.linter_run_stub import build_project_run_linter + + +class _Master: + def __init__(self, report_folder): + self.report_folder = report_folder + + +def _make_linter( + workspace, report_folder, request_id, ignore_file=None, user_args=None +): + linter = SecretLintLinter.__new__(SecretLintLinter) + linter.linter_name = "secretlint" + linter.workspace = workspace + linter.report_folder = report_folder + linter.request_id = request_id + linter.ignore_file = ignore_file + linter.cli_lint_user_args = user_args if user_args is not None else [] + linter.master = _Master(report_folder) + linter.megalinter_ignore_file = None + linter.ignore_tmp_folder = None + return linter + + +class SecretLintLinterTest(IsolatedConfigTestCase): + def test_setup_isolates_ambient_megalinter_config(self): + # Without the patch.dict(..., clear=True) guard in setUp, an ambient + # MEGALINTER_CONFIG pointing at a non-existent file reaches + # workspace + os.path.sep + config_file_name with workspace=None, raising + # TypeError before config.get is ever reached. + with patch.dict(os.environ, {"MEGALINTER_CONFIG": "no-such-config.yml"}): + self.setUp() + + self.assertNotEqual( + config.get(self.request_id, "MEGALINTER_CONFIG", None), + "no-such-config.yml", + ) + + def test_ignore_file_written_with_relative_path(self): + with tempfile.TemporaryDirectory() as workspace: + report_folder = os.path.join(workspace, "megalinter-reports") + linter = _make_linter(workspace, report_folder, self.request_id) + + relative_path = linter.build_megalinter_ignore_file() + + self.assertEqual( + relative_path, f"megalinter-reports/{MEGALINTER_IGNORE_FILE_NAME}" + ) + self.assertFalse(os.path.isabs(relative_path)) + self.assertTrue( + os.path.isfile(os.path.join(report_folder, MEGALINTER_IGNORE_FILE_NAME)) + ) + + def test_report_folder_is_excluded(self): + with tempfile.TemporaryDirectory() as workspace: + report_folder = os.path.join(workspace, "megalinter-reports") + linter = _make_linter(workspace, report_folder, self.request_id) + + linter.build_megalinter_ignore_file() + + with open( + os.path.join(report_folder, MEGALINTER_IGNORE_FILE_NAME), + "r", + encoding="utf-8", + ) as fh: + content = fh.read() + # Bare pattern, not megalinter-reports/** : node-ignore prunes the + # directory outright rather than testing each child + self.assertIn("\nmegalinter-reports\n", content) + self.assertNotIn("megalinter-reports/**", content) + # Only the report folder is added. MegaLinter's wider excluded + # directories must NOT be, because secretlint is a secrets scanner and + # build artifacts such as .terraform or .terragrunt-cache can contain + # real credentials baked in at generate time + self.assertNotIn(".terraform", content) + self.assertNotIn(".terragrunt-cache", content) + + def test_user_patterns_are_merged_before_megalinter_patterns(self): + with tempfile.TemporaryDirectory() as workspace: + report_folder = os.path.join(workspace, "megalinter-reports") + user_ignore = os.path.join(workspace, ".secretlintignore") + with open(user_ignore, "w", encoding="utf-8") as fh: + fh.write("my/fixtures\n!megalinter-reports/keep.json\n") + linter = _make_linter( + workspace, report_folder, self.request_id, ignore_file=user_ignore + ) + + linter.build_megalinter_ignore_file() + + with open( + os.path.join(report_folder, MEGALINTER_IGNORE_FILE_NAME), + "r", + encoding="utf-8", + ) as fh: + content = fh.read() + self.assertIn("my/fixtures", content) + # gitignore semantics are last match wins, so MegaLinter's patterns must + # come after the user's negation to stay authoritative + self.assertLess( + content.index("!megalinter-reports/keep.json"), + content.index("\nmegalinter-reports\n"), + ) + + def test_unreadable_source_ignore_file_degrades(self): + # before_lint_files is called from Linter.run with no exception boundary up + # to megalinter/run.py, so raising here would abort every linter, not just + # secretlint. A non-UTF-8 .gitignore must therefore degrade, never raise. + with tempfile.TemporaryDirectory() as workspace: + report_folder = os.path.join(workspace, "megalinter-reports") + with open(os.path.join(workspace, ".gitignore"), "wb") as fh: + fh.write(b"\xff\xfe invalid\n") + linter = _make_linter(workspace, report_folder, self.request_id) + + relative_path = linter.build_megalinter_ignore_file() + + self.assertEqual( + relative_path, f"megalinter-reports/{MEGALINTER_IGNORE_FILE_NAME}" + ) + generated_file = os.path.join(report_folder, MEGALINTER_IGNORE_FILE_NAME) + self.assertTrue(os.path.isfile(generated_file)) + with open(generated_file, "r", encoding="utf-8") as fh: + content = fh.read() + self.assertIn("\nmegalinter-reports\n", content) + + def test_write_failure_logs_warning_with_strerror_and_returns_none(self): + # A real write failure, not the megalinter_ignore_file = None shortcut used + # by test_get_ignore_arguments_falls_back_to_ignore_file_base_name, so this + # actually exercises the except OSError branch of + # build_megalinter_ignore_file rather than assuming its postcondition. + # The denial is injected instead of produced with os.chmod(0o500) because + # MegaLinter's test image runs as root, and root bypasses directory + # permissions: the write would succeed and the branch never run. The error + # carries an errno, a strerror and a filename like a kernel-raised one, as a + # bare PermissionError("denied") has strerror None and would satisfy the + # "(None)" assertion below for the wrong reason. + with tempfile.TemporaryDirectory() as workspace: + report_folder = os.path.join(workspace, "megalinter-reports") + os.makedirs(report_folder) + linter = _make_linter(workspace, report_folder, self.request_id) + ignore_file_path = os.path.join(report_folder, MEGALINTER_IGNORE_FILE_NAME) + real_open = builtins.open + + # Only the generated ignore file is denied: every other open, such as + # the source ignore file this method also reads, must behave normally + def deny_ignore_file_write(file, *args, **kwargs): + if file == ignore_file_path: + raise PermissionError(errno.EACCES, os.strerror(errno.EACCES), file) + return real_open(file, *args, **kwargs) + + with patch("builtins.open", deny_ignore_file_write): + with self.assertLogs(level="WARNING") as logs: + relative_path = linter.build_megalinter_ignore_file() + + self.assertIsNone(relative_path) + self.assertFalse(os.path.exists(ignore_file_path)) + warning_messages = [ + message for message in logs.output if "Unable to write" in message + ] + self.assertEqual(len(warning_messages), 1) + self.assertNotIn("(None)", warning_messages[0]) + self.assertIn(os.strerror(errno.EACCES), warning_messages[0]) + + def test_gitignore_used_when_no_secretlintignore(self): + with tempfile.TemporaryDirectory() as workspace: + report_folder = os.path.join(workspace, "megalinter-reports") + with open( + os.path.join(workspace, ".gitignore"), "w", encoding="utf-8" + ) as fh: + fh.write("dist\n") + linter = _make_linter(workspace, report_folder, self.request_id) + + linter.build_megalinter_ignore_file() + + with open( + os.path.join(report_folder, MEGALINTER_IGNORE_FILE_NAME), + "r", + encoding="utf-8", + ) as fh: + content = fh.read() + self.assertIn("dist", content) + + def _generated_content_for_user_ignore_arg(self, workspace, user_args): + report_folder = os.path.join(workspace, "megalinter-reports") + with open( + os.path.join(workspace, "custom-ignore"), "w", encoding="utf-8" + ) as fh: + fh.write("vendor\n") + linter = _make_linter( + workspace, report_folder, self.request_id, user_args=user_args + ) + + linter.build_megalinter_ignore_file() + + with open( + os.path.join(report_folder, MEGALINTER_IGNORE_FILE_NAME), + "r", + encoding="utf-8", + ) as fh: + return fh.read() + + def test_user_supplied_ignore_arg_is_merged(self): + # secretlint takes the last --secretlintignore and drops the earlier one, so + # a user file that is not merged would stop being applied entirely + with tempfile.TemporaryDirectory() as workspace: + content = self._generated_content_for_user_ignore_arg( + workspace, ["--secretlintignore", "custom-ignore"] + ) + self.assertIn("vendor", content) + + def test_custom_report_folder_is_excluded(self): + with tempfile.TemporaryDirectory() as workspace: + report_folder = os.path.join(workspace, "build", "ml-reports") + linter = _make_linter(workspace, report_folder, self.request_id) + + relative_path = linter.build_megalinter_ignore_file() + + self.assertEqual( + relative_path, f"build/ml-reports/{MEGALINTER_IGNORE_FILE_NAME}" + ) + with open( + os.path.join(report_folder, MEGALINTER_IGNORE_FILE_NAME), + "r", + encoding="utf-8", + ) as fh: + content = fh.read() + self.assertIn("build/ml-reports", content) + + def test_get_ignore_arguments_uses_generated_file(self): + with tempfile.TemporaryDirectory() as workspace: + report_folder = os.path.join(workspace, "megalinter-reports") + linter = _make_linter(workspace, report_folder, self.request_id) + linter.megalinter_ignore_file = ( + f"megalinter-reports/{MEGALINTER_IGNORE_FILE_NAME}" + ) + + self.assertEqual( + linter.get_ignore_arguments([]), + [ + "--secretlintignore", + f"megalinter-reports/{MEGALINTER_IGNORE_FILE_NAME}", + ], + ) + + def test_report_folder_outside_workspace_adds_no_parent_pattern(self): + with tempfile.TemporaryDirectory() as root: + workspace = os.path.join(root, "workspace") + report_folder = os.path.join(root, "reports") + os.makedirs(workspace) + linter = _make_linter(workspace, report_folder, self.request_id) + + relative_path = linter.build_megalinter_ignore_file() + + # secretlint resolves this with path.join, which normalizes "..", so the + # generated file is still found + self.assertEqual(relative_path, f"../reports/{MEGALINTER_IGNORE_FILE_NAME}") + generated_file = os.path.join(report_folder, MEGALINTER_IGNORE_FILE_NAME) + self.assertTrue(os.path.isfile(generated_file)) + with open(generated_file, "r", encoding="utf-8") as fh: + content = fh.read() + # "../reports" is not a valid gitignore pattern, and a report folder + # outside the workspace is never scanned anyway + self.assertNotIn("..", content) + + def test_get_ignore_arguments_falls_back_to_ignore_file_base_name(self): + # secretlint resolves --secretlintignore as path.join(cwd, value) and + # silently drops an absolute path, so the fallback used when MegaLinter + # cannot write its own ignore file must pass the base name + with tempfile.TemporaryDirectory() as workspace: + report_folder = os.path.join(workspace, "megalinter-reports") + ignore_file = os.path.join( + workspace, ".github", "linters", ".secretlintignore" + ) + linter = _make_linter( + workspace, report_folder, self.request_id, ignore_file=ignore_file + ) + linter.cli_lint_ignore_arg_name = "--secretlintignore" + linter.cli_lint_extra_args_after = [] + + self.assertIsNone(linter.megalinter_ignore_file) + self.assertEqual( + linter.get_ignore_arguments([]), + ["--secretlintignore", ".secretlintignore"], + ) + + def _resolve_ignore_path_and_register_cleanup(self, workspace, relative_path): + # secretlint resolves --secretlintignore as path.join(cwd, value) with cwd + # being the workspace, so this is the contract that actually matters at + # runtime: the returned value must combine with the workspace to reach the + # file secretlint will actually read. + resolved_path = os.path.normpath(os.path.join(workspace, relative_path)) + self.addCleanup( + shutil.rmtree, os.path.dirname(resolved_path), ignore_errors=True + ) + return resolved_path + + def _make_reports_disabled_linter_with_user_ignore(self, workspace): + user_ignore = os.path.join(workspace, ".secretlintignore") + with open(user_ignore, "w", encoding="utf-8") as fh: + fh.write("my/fixtures\n") + return _make_linter(workspace, "none", self.request_id, ignore_file=user_ignore) + + def _generated_content_at(self, workspace, relative_path): + resolved_path = self._resolve_ignore_path_and_register_cleanup( + workspace, relative_path + ) + with open(resolved_path, "r", encoding="utf-8") as fh: + return fh.read() + + def test_reports_disabled_writes_to_temp_folder(self): + # _make_linter already wires master.report_folder to "none", which is + # what can_write_report_files checks. A .secretlintignore living in + # LINTER_RULES_PATH must still be applied even though there is no report + # folder to write the generated file into (issue #3979 review finding). + with tempfile.TemporaryDirectory() as workspace: + linter = _make_linter(workspace, "none", self.request_id) + + relative_path = linter.build_megalinter_ignore_file() + + self.assertIsNotNone(relative_path) + self.assertFalse(os.path.isabs(relative_path)) + resolved_path = self._resolve_ignore_path_and_register_cleanup( + workspace, relative_path + ) + self.assertTrue(os.path.isfile(resolved_path)) + + def test_reports_disabled_merges_user_patterns(self): + with tempfile.TemporaryDirectory() as workspace: + linter = self._make_reports_disabled_linter_with_user_ignore(workspace) + + relative_path = linter.build_megalinter_ignore_file() + + self.assertIn( + "my/fixtures", self._generated_content_at(workspace, relative_path) + ) + + def test_reports_disabled_adds_no_report_folder_pattern(self): + with tempfile.TemporaryDirectory() as workspace: + linter = self._make_reports_disabled_linter_with_user_ignore(workspace) + + content = self._generated_content_at( + workspace, linter.build_megalinter_ignore_file() + ) + + self.assertIn("my/fixtures", content) + self.assertNotIn( + "Added by MegaLinter: never lint MegaLinter's own output", content + ) + + def test_reports_disabled_logs_warning_naming_report_output_folder(self): + # The warning only fires when reports being disabled actually put + # something at risk: a user ignore file must be present + with tempfile.TemporaryDirectory() as workspace: + linter = self._make_reports_disabled_linter_with_user_ignore(workspace) + + with self.assertLogs(level="WARNING") as logs: + relative_path = linter.build_megalinter_ignore_file() + + self.assertTrue( + any("REPORT_OUTPUT_FOLDER" in message for message in logs.output) + ) + self._resolve_ignore_path_and_register_cleanup(workspace, relative_path) + + def test_reports_disabled_no_source_ignore_file_logs_no_warning(self): + # No ignore file anywhere means nothing was ever at risk of being + # dropped, so the noisy REPORT_OUTPUT_FOLDER warning must not fire + with tempfile.TemporaryDirectory() as workspace: + linter = _make_linter(workspace, "none", self.request_id) + + with self.assertNoLogs(level="WARNING"): + relative_path = linter.build_megalinter_ignore_file() + + self._resolve_ignore_path_and_register_cleanup(workspace, relative_path) + + def test_reports_disabled_uses_gitignore_when_no_secretlintignore(self): + with tempfile.TemporaryDirectory() as workspace: + with open( + os.path.join(workspace, ".gitignore"), "w", encoding="utf-8" + ) as fh: + fh.write("dist\n") + linter = _make_linter(workspace, "none", self.request_id) + + relative_path = linter.build_megalinter_ignore_file() + + self.assertIn("dist", self._generated_content_at(workspace, relative_path)) + + def _make_run_ready_linter(self, workspace): + linter = _make_linter(workspace, "none", self.request_id) + build_project_run_linter( + self.request_id, + linter, + name="TEST_SECRETLINT", + output_sarif=False, + ) + return linter + + def _assert_ignore_tmp_folder_cleaned(self, linter, workspace): + self.assertIsNone(linter.ignore_tmp_folder) + resolved_path = os.path.normpath( + os.path.join(workspace, linter.megalinter_ignore_file) + ) + self.assertFalse(os.path.isdir(os.path.dirname(resolved_path))) + + def test_run_cleans_up_ignore_tmp_folder(self): + # A leaked temp directory is worth removing on every invocation regardless + # of process lifetime: nothing else ever cleans up the folder created by + # build_megalinter_ignore_file when REPORT_OUTPUT_FOLDER is disabled + with tempfile.TemporaryDirectory() as workspace: + linter = self._make_run_ready_linter(workspace) + + with patch.object(Linter, "process_linter", return_value=(0, "")): + linter.run() + + self._assert_ignore_tmp_folder_cleaned(linter, workspace) + + def test_run_cleans_up_ignore_tmp_folder_when_process_linter_raises(self): + # Proves the "finally" in SecretLintLinter.run() matters: a plain + # self.remove_ignore_tmp_folder() placed after super().run() with no + # try/finally would pass the happy-path test above just as well, but + # would leak the temp folder whenever process_linter raises + with tempfile.TemporaryDirectory() as workspace: + linter = self._make_run_ready_linter(workspace) + + with patch.object( + Linter, "process_linter", side_effect=RuntimeError("boom") + ): + with self.assertRaises(RuntimeError): + linter.run() + + self._assert_ignore_tmp_folder_cleaned(linter, workspace) + + def test_user_supplied_ignore_arg_with_equals_form_is_merged(self): + with tempfile.TemporaryDirectory() as workspace: + content = self._generated_content_for_user_ignore_arg( + workspace, ["--secretlintignore=custom-ignore"] + ) + self.assertIn("vendor", content) + + def test_user_supplied_ignore_arg_not_a_file_logs_warning_and_degrades(self): + with tempfile.TemporaryDirectory() as workspace: + report_folder = os.path.join(workspace, "megalinter-reports") + linter = _make_linter( + workspace, + report_folder, + self.request_id, + user_args=["--secretlintignore", "does-not-exist"], + ) + + with self.assertLogs(level="WARNING") as logs: + source_ignore_file = linter.get_source_ignore_file() + + self.assertIsNone(source_ignore_file) + self.assertTrue( + any( + "does-not-exist" in message and "not" in message + for message in logs.output + ) + ) + + def test_unresolvable_user_ignore_arg_does_not_fall_back_to_gitignore(self): + # A .gitignore in a real repository routinely lists exactly what a secrets + # scanner exists to find (.env, *.pem, credentials.json, secrets/). An + # unresolvable --secretlintignore must not silently promote those patterns + # to authoritative status via the fallback chain. + with tempfile.TemporaryDirectory() as workspace: + report_folder = os.path.join(workspace, "megalinter-reports") + with open( + os.path.join(workspace, ".gitignore"), "w", encoding="utf-8" + ) as fh: + fh.write(".env\n") + linter = _make_linter( + workspace, + report_folder, + self.request_id, + user_args=["--secretlintignore", "does-not-exist"], + ) + + with self.assertLogs(level="WARNING"): + source_ignore_file = linter.get_source_ignore_file() + + self.assertIsNone(source_ignore_file) diff --git a/megalinter/tests/test_megalinter/utils_test.py b/megalinter/tests/test_megalinter/utils_test.py index 9a774b5c934..679153bac87 100644 --- a/megalinter/tests/test_megalinter/utils_test.py +++ b/megalinter/tests/test_megalinter/utils_test.py @@ -4,8 +4,12 @@ """ +import os import unittest +import uuid +from unittest.mock import patch +from megalinter import config, utils from megalinter.logger import fetch_betterleaks_regexes, sanitize_string @@ -30,3 +34,44 @@ def test_fetch_betterleaks_regexes(self): regexes = fetch_betterleaks_regexes() self.assertIsInstance(regexes, list, "Regexes should be a list") self.assertGreater(len(regexes), 0, "Regexes list should not be empty") + + def test_report_folder_excluded_by_default(self): + # init_config(request_id, workspace=None) copies os.environ wholesale into + # the config when no workspace is given, so REPORT_OUTPUT_FOLDER, + # EXCLUDED_DIRECTORIES or MEGALINTER_CONFIG set in the ambient environment + # (plausible when running inside a MegaLinter container) would otherwise + # leak into this test + request_id = str(uuid.uuid1()) + with patch.dict(os.environ, {}, clear=True): + config.init_config(request_id) + try: + excluded = utils.get_excluded_directories(request_id) + self.assertIn("megalinter-reports", excluded) + finally: + config.delete(request_id) + + def test_report_folder_excluded_despite_excluded_directories_override(self): + # EXCLUDED_DIRECTORIES replaces the defaults, but MegaLinter's own output + # folder must never become lintable + request_id = str(uuid.uuid1()) + with patch.dict(os.environ, {}, clear=True): + config.init_config(request_id) + try: + config.set_value(request_id, "EXCLUDED_DIRECTORIES", ["only_this"]) + excluded = utils.get_excluded_directories(request_id) + self.assertIn("only_this", excluded) + self.assertIn("megalinter-reports", excluded) + finally: + config.delete(request_id) + + def test_custom_report_folder_excluded_despite_override(self): + request_id = str(uuid.uuid1()) + with patch.dict(os.environ, {}, clear=True): + config.init_config(request_id) + try: + config.set_value(request_id, "REPORT_OUTPUT_FOLDER", "build/ml-reports") + config.set_value(request_id, "EXCLUDED_DIRECTORIES", ["only_this"]) + excluded = utils.get_excluded_directories(request_id) + self.assertIn("build/ml-reports", excluded) + finally: + config.delete(request_id) diff --git a/megalinter/utils.py b/megalinter/utils.py index 7622dd913db..ac7adfd8a82 100644 --- a/megalinter/utils.py +++ b/megalinter/utils.py @@ -125,6 +125,9 @@ def get_excluded_directories(request_id): cached = _excluded_directories_cache.get(cache_key) if cached is not None: return cached + report_output_folder = config.get( + request_id, "REPORT_OUTPUT_FOLDER", "megalinter-reports" + ) default_excluded_dirs = [ "__pycache__", ".git", @@ -136,13 +139,17 @@ def get_excluded_directories(request_id): ".terraform", ".terragrunt-cache", "node_modules", - config.get(request_id, "REPORT_OUTPUT_FOLDER", "megalinter-reports"), + report_output_folder, ] excluded_dirs = config.get_list( request_id, "EXCLUDED_DIRECTORIES", default_excluded_dirs ) excluded_dirs += config.get_list(request_id, "ADDITIONAL_EXCLUDED_DIRECTORIES", []) result = set(excluded_dirs) + # Added after the EXCLUDED_DIRECTORIES override so MegaLinter's own output folder + # cannot be configured away: linting it is never a valid outcome, and doing so + # races with the reporters still writing into it + result.add(report_output_folder) _excluded_directories_cache[cache_key] = result return result