diff --git a/master_changes.md b/master_changes.md index 1c79abf3981..8c58c331efd 100644 --- a/master_changes.md +++ b/master_changes.md @@ -72,6 +72,7 @@ users) ## Sandbox ## VCS + * Disable git gc/maintenance on repositories opam maintains [#7073 @kit-ty-kate - partially fix #7031] ## Build @@ -134,3 +135,5 @@ users) ## opam-format ## opam-core + * `OpamStd.Env`: Add the `raw = string array` type alias [#7073 @kit-ty-kate] + * `OpamStd.Env`: Add the `get_from_array` and `add_and_replace` functions [#7073 @kit-ty-kate] diff --git a/src/core/opamStd.ml b/src/core/opamStd.ml index b5ea475ddb3..bd62636f974 100644 --- a/src/core/opamStd.ml +++ b/src/core/opamStd.ml @@ -741,6 +741,46 @@ module Env = struct module Map = Map.Make(M) end + type raw = string array + + let raw_env = Unix.environment + + let env_starts_with = + if Sys.win32 then + fun ~prefix x -> + let prefix = String.lowercase_ascii prefix in + let x = String.lowercase_ascii x in + OpamCompat.String.starts_with ~prefix x + else + OpamCompat.String.starts_with + + let get_from_array env key = + let prefix = key ^ "=" in + let prefix_len = String.length prefix in + let exception Found of string in + try + for i = 0 to Array.length env - 1 do + let x = env.(i) in + if env_starts_with ~prefix x then + raise_notrace (Found x) + done; + None + with Found x -> + Some (String.sub x prefix_len (String.length x - prefix_len)) + + let add_and_replace env (key, value) = + let prefix = key ^ "=" in + let elem = prefix ^ value in + (* We always append [elem] to the environment instead of setting inplace + to avoid the double copy as we need to copy once anyway *) + let env = Array.append env [|elem|] in + for i = 0 to Array.length env - 2 do + let x = env.(i) in + if env_starts_with ~prefix x then + env.(i) <- elem; + done; + env + let to_list env = List.rev_map (fun s -> match OpamString.cut_at s '=' with @@ -748,8 +788,6 @@ module Env = struct | Some p -> p) (Array.to_list env) - let raw_env = Unix.environment - let list = let lazy_env = lazy (to_list (raw_env ())) in fun () -> Lazy.force lazy_env diff --git a/src/core/opamStd.mli b/src/core/opamStd.mli index d219de17640..d99967cdf77 100644 --- a/src/core/opamStd.mli +++ b/src/core/opamStd.mli @@ -373,6 +373,28 @@ end module Env : sig + (** {3 Raw array functions} *) + + type raw = string array + + (** Same as {!Unix.environment} *) + val raw_env: unit -> raw + + (** [get_from_array array key] returns the value of the first element + of [array] associated with [key]. Returns [None] if nothing was found. *) + val get_from_array: raw -> string -> string option + + (** [add_and_replace array (key, value)] returns a fresh array containing + at least [key=value] and [array] where all the elements matching [key] + will be replaced by [key=value] as well. + + This makes sure any applications will always associate [key] with [value] + as per POSIX.1-2024: + > If more than one string in an environment of a process has the same name, + > the consequences are undefined. + https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/V1_chap08.html *) + val add_and_replace: raw -> (string * string) -> raw + (** {3 Generic functions} *) (** Remove from a c-separated list of string the ones with the given prefix *) @@ -436,7 +458,6 @@ module Env : sig val getopt_full: Name.t -> Name.t * string option val list: unit -> (Name.t * string) list - val raw_env: unit -> string array (** [cyg_env ~env ~cygbin ~git_location] returns [env] environment with its PATH variable updated with [git_location] and [cygbin] at the beginning, diff --git a/src/repository/opamGit.ml b/src/repository/opamGit.ml index d2162d58a7e..ee7a392cecc 100644 --- a/src/repository/opamGit.ml +++ b/src/repository/opamGit.ml @@ -14,13 +14,49 @@ open OpamProcess.Job.Op (* let log fmt = OpamConsole.log "GIT" fmt *) -let git_env = [| - "GIT_CONFIG_GLOBAL="^Filename.null; - "GIT_CONFIG_SYSTEM="^Filename.null; -|] +let git_env = [ + "GIT_CONFIG_GLOBAL", Filename.null; + "GIT_CONFIG_SYSTEM", Filename.null; +] -let env () = - Array.append git_env (OpamProcess.default_env ()) +let git_config = [ + "gc.autoDetach", "false"; + "maintenance.autoDetach", "false"; +] + +let env = + let env = lazy begin + let env = OpamProcess.default_env () in + let env = List.fold_left OpamStd.Env.add_and_replace env git_env in + let config_count_name = "GIT_CONFIG_COUNT" in + let old_config_count = + let open OpamStd.Option.Op in + (OpamStd.Env.get_from_array env config_count_name >>= int_of_string_opt) + +! 0 + in + let config_key_prefix = "GIT_CONFIG_KEY_" in + let config_value_prefix = "GIT_CONFIG_VALUE_" in + let new_config_count = old_config_count + List.length git_config in + let env = + OpamStd.Env.add_and_replace env + (config_count_name, Int.to_string new_config_count) + in + let rec aux env n = function + | [] -> env + | (key, value)::xs -> + let env = + OpamStd.Env.add_and_replace env + (config_key_prefix ^ Int.to_string n, key) + in + let env = + OpamStd.Env.add_and_replace env + (config_value_prefix ^ Int.to_string n, value) + in + aux env (n + 1) xs + in + aux env old_config_count git_config + end in + fun () -> Lazy.force env module VCS : OpamVCS.VCS = struct @@ -45,6 +81,12 @@ module VCS : OpamVCS.VCS = struct OpamFilename.mkdir repo_root; OpamProcess.Job.of_list [ git repo_root [ "init"; "--initial-branch=main" ]; + (* Disable detachable jobs to avoid race conditions. + For example while building a package, if the user build steps + calls git they might end up spawning a maintenance job which would + then interfere with the removal of the build directory later. *) + git repo_root [ "config" ; "--local" ; "gc.autoDetach"; "false"]; + git repo_root [ "config" ; "--local" ; "maintenance.autoDetach"; "false"]; (* Enforce this option, it can break our use of git if set *) git repo_root [ "config" ; "--local" ; "fetch.prune"; "false"]; (* We reset diff.noprefix to ensure we get a `-p1` patch and avoid . *) diff --git a/tests/reftests/action-disk.test b/tests/reftests/action-disk.test index acd00130160..0ed8fe1b27c 100644 --- a/tests/reftests/action-disk.test +++ b/tests/reftests/action-disk.test @@ -1271,6 +1271,8 @@ SYSTEM mkdir ${BASEDIR}/OPAM/install-from-git-pin-all/. Processing 1/1: [main-gpin: git] + git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin" "init" "--initial-branch=main" - Initialized empty Git repository in ${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin/.git/ ++ git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin" "config" "--local" "gc.autoDetach" "false" ++ git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin" "config" "--local" "maintenance.autoDetach" "false" + git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin" "config" "--local" "fetch.prune" "false" + git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin" "config" "--local" "diff.noprefix" "false" + git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin" "config" "--local" "core.autocrlf" "false" @@ -1630,6 +1632,8 @@ FILE(switch-config) Wrote ${BASEDIR}/OPAM/install-from-git-pin-all/. FILE(switch-state) Wrote ${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/backup/state-today.export atomically in 0.000s + git "-C" "${BASEDIR}/main-gpin" "symbolic-ref" "--quiet" "--short" "HEAD" + git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin" "init" "--initial-branch=main" ++ git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin" "config" "--local" "gc.autoDetach" "false" ++ git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin" "config" "--local" "maintenance.autoDetach" "false" + git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin" "config" "--local" "fetch.prune" "false" + git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin" "config" "--local" "diff.noprefix" "false" + git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin" "config" "--local" "core.autocrlf" "false" @@ -1845,6 +1849,8 @@ SYSTEM mkdir ${OPAMTMP} SYSTEM mkdir ${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin.dev Processing 1/2: [main-gpin.dev: git] + git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin.dev" "init" "--initial-branch=main" ++ git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin.dev" "config" "--local" "gc.autoDetach" "false" ++ git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin.dev" "config" "--local" "maintenance.autoDetach" "false" + git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin.dev" "config" "--local" "fetch.prune" "false" + git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin.dev" "config" "--local" "diff.noprefix" "false" + git "-C" "${BASEDIR}/OPAM/install-from-git-pin-all/.opam-switch/sources/main-gpin.dev" "config" "--local" "core.autocrlf" "false" diff --git a/tests/reftests/download.test b/tests/reftests/download.test index d967cef5b60..713003e1332 100644 --- a/tests/reftests/download.test +++ b/tests/reftests/download.test @@ -237,6 +237,8 @@ SYSTEM mkdir ${OPAMTMP} SYSTEM mkdir ${BASEDIR}/OPAM/download/.opam-switch/sources/bar.1 Processing 1/1: [bar.1: git] + git "-C" "${BASEDIR}/OPAM/download/.opam-switch/sources/bar.1" "init" "--initial-branch=main" ++ git "-C" "${BASEDIR}/OPAM/download/.opam-switch/sources/bar.1" "config" "--local" "gc.autoDetach" "false" ++ git "-C" "${BASEDIR}/OPAM/download/.opam-switch/sources/bar.1" "config" "--local" "maintenance.autoDetach" "false" + git "-C" "${BASEDIR}/OPAM/download/.opam-switch/sources/bar.1" "config" "--local" "fetch.prune" "false" + git "-C" "${BASEDIR}/OPAM/download/.opam-switch/sources/bar.1" "config" "--local" "diff.noprefix" "false" + git "-C" "${BASEDIR}/OPAM/download/.opam-switch/sources/bar.1" "config" "--local" "core.autocrlf" "false" @@ -296,6 +298,8 @@ SYSTEM mkdir ${OPAMTMP} SYSTEM mkdir ${BASEDIR}/OPAM/download/.opam-switch/sources/qux.1 Processing 1/1: [qux.1: git] + git "-C" "${BASEDIR}/OPAM/download/.opam-switch/sources/qux.1" "init" "--initial-branch=main" ++ git "-C" "${BASEDIR}/OPAM/download/.opam-switch/sources/qux.1" "config" "--local" "gc.autoDetach" "false" ++ git "-C" "${BASEDIR}/OPAM/download/.opam-switch/sources/qux.1" "config" "--local" "maintenance.autoDetach" "false" + git "-C" "${BASEDIR}/OPAM/download/.opam-switch/sources/qux.1" "config" "--local" "fetch.prune" "false" + git "-C" "${BASEDIR}/OPAM/download/.opam-switch/sources/qux.1" "config" "--local" "diff.noprefix" "false" + git "-C" "${BASEDIR}/OPAM/download/.opam-switch/sources/qux.1" "config" "--local" "core.autocrlf" "false" diff --git a/tests/reftests/repository.test b/tests/reftests/repository.test index 814ea17b494..ff052927bdc 100644 --- a/tests/reftests/repository.test +++ b/tests/reftests/repository.test @@ -1060,6 +1060,8 @@ REPOSITORY update changes from git+file://${BASEDIR}/CHG Processing 1/1: [changes: git] + git "-C" "${BASEDIR}/OPAM/repo/changes" "init" "--initial-branch=main" - Initialized empty Git repository in ${BASEDIR}/OPAM/repo/changes/.git/ ++ git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "gc.autoDetach" "false" ++ git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "maintenance.autoDetach" "false" + git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "fetch.prune" "false" + git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "diff.noprefix" "false" + git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "core.autocrlf" "false" @@ -1114,6 +1116,8 @@ REPOSITORY update changes from git+file://${BASEDIR}/CHG Processing 1/1: [changes: git] + git "-C" "${BASEDIR}/OPAM/repo/changes" "init" "--initial-branch=main" - Initialized empty Git repository in ${BASEDIR}/OPAM/repo/changes/.git/ ++ git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "gc.autoDetach" "false" ++ git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "maintenance.autoDetach" "false" + git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "fetch.prune" "false" + git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "diff.noprefix" "false" + git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "core.autocrlf" "false" @@ -1236,6 +1240,8 @@ REPOSITORY update changes from git+file://${BASEDIR}/CHG Processing 1/1: [changes: git] + git "-C" "${BASEDIR}/OPAM/repo/changes" "init" "--initial-branch=main" - Initialized empty Git repository in ${BASEDIR}/OPAM/repo/changes/.git/ ++ git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "gc.autoDetach" "false" ++ git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "maintenance.autoDetach" "false" + git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "fetch.prune" "false" + git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "diff.noprefix" "false" + git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "core.autocrlf" "false" @@ -1287,6 +1293,8 @@ REPOSITORY update changes from git+file://${BASEDIR}/CHG Processing 1/1: [changes: git] + git "-C" "${BASEDIR}/OPAM/repo/changes" "init" "--initial-branch=main" - Initialized empty Git repository in ${BASEDIR}/OPAM/repo/changes/.git/ ++ git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "gc.autoDetach" "false" ++ git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "maintenance.autoDetach" "false" + git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "fetch.prune" "false" + git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "diff.noprefix" "false" + git "-C" "${BASEDIR}/OPAM/repo/changes" "config" "--local" "core.autocrlf" "false"