[10.0.4xx] Use dotnetup preview for .NET acquisition

.gitignore

@@ -40,6 +40,9 @@ cmake/
 # Helix payload
 .dotnet.payload
 
+# dotnetup build output
+eng/dotnetup/
+
 # MSBuild Logs
 **/MSBuild_Logs/MSBuild_pid-*.failure.txt
 

eng/configure-toolset.ps1

@@ -4,3 +4,62 @@
 
 $script:useInstalledDotNetCli = $false
 
+# Shared dotnetup acquisition helpers (architecture detection, cache freshness, download).
+. (Join-Path $PSScriptRoot 'dotnetup-shared.ps1')
+
+# Pre-install the bootstrap SDK pinned in global.json using dotnetup into the
+# repo-local .dotnet directory that arcade's InitializeDotnetCli will pick up.
+#
+# Skipped during VMR / source-build (no network) and when -restore was not requested.
+function InstallBootstrapSdkWithDotnetup() {
+    $dotnetSdkVersion = $GlobalJson.tools.dotnet
+    if ((-not $restore) -or $fromVmr -or [string]::IsNullOrEmpty($dotnetSdkVersion)) {
+        return
+    }
+
+    # Collect all SDK versions to install (primary + any additional).
+    $sdkVersions = @($dotnetSdkVersion)
+    if ($GlobalJson.tools.PSObject.Properties['additionalDotNetVersions']) {
+        $sdkVersions += @($GlobalJson.tools.additionalDotNetVersions | Where-Object { -not [string]::IsNullOrEmpty($_) })
+    }
+
+    $dotnetRoot = Join-Path $RepoRoot '.dotnet'
+
+    Write-Host "Installing SDK(s) '$($sdkVersions -join ', ')' to '$dotnetRoot' via dotnetup..." -ForegroundColor Cyan
+
+    $dotnetupDir = Join-Path $PSScriptRoot 'dotnetup'
+    $dotnetupExe = Join-Path $dotnetupDir (GetExecutableFileName 'dotnetup')
+
+    if (-not (Test-ShouldUseCachedDotnetup $dotnetupExe)) {
+        try {
+            Install-DotnetupFromAkaMs $dotnetupDir
+        }
+        catch {
+            Write-Host "Failed to acquire dotnetup: $($_.Exception.Message). Will fall back to standard dotnet-install script." -ForegroundColor Yellow
+            return
+        }
+    }
+
+    # Keep dotnetup's manifest under artifacts instead of the user's home dir.
+    $env:DOTNET_DOTNETUP_DATA_DIR = Join-Path $ArtifactsDir '.dotnetup'
+
+    if (-not (Test-Path Variable:LASTEXITCODE)) { $global:LASTEXITCODE = 0 }
+    $installExitCode = Invoke-DotnetupNativeCommand {
+        & $dotnetupExe sdk install @sdkVersions `
+            --install-path $dotnetRoot `
+            --untracked `
+            --set-default-install false `
+            --interactive false
+    }
+    if ($installExitCode -ne 0) {
+        Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install .NET SDK(s) '$($sdkVersions -join ', ')' to '$dotnetRoot' using dotnetup (exit code '$installExitCode'). Will fall back to standard dotnet-install script."
+        return
+    }
+
+    # Record the installed SDK so CleanOutStage0ToolsetsAndRuntimes does not
+    # later treat this install as stale and wipe it (forcing a build rerun).
+    Set-Content -Path (Join-Path $dotnetRoot '.version') -Value $dotnetSdkVersion -NoNewline
+}
+
+InstallBootstrapSdkWithDotnetup
+

eng/configure-toolset.sh

@@ -4,5 +4,79 @@
 
 useInstalledDotNetCli="false"
 
+# Shared dotnetup acquisition helpers (architecture detection, cache freshness, download).
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/dotnetup-shared.sh"
+
+# Pre-install the bootstrap SDK pinned in global.json using dotnetup into the
+# repo-local .dotnet directory that arcade's InitializeDotNetCli will pick up.
+#
+# Skipped during VMR / source-build (no network) and when --restore was not requested.
+function InstallBootstrapSdkWithDotnetup {
+  ReadGlobalVersion "dotnet"
+  local dotnet_sdk_version=$_ReadGlobalVersion
+  if [[ "$restore" != true || "$from_vmr" == true || -z "$dotnet_sdk_version" ]]; then
+    return
+  fi
+
+  # Collect all SDK versions to install (primary + any additional).
+  local sdk_versions=("$dotnet_sdk_version")
+  # Extract additionalDotNetVersions from global.json using grep/sed (no python dependency).
+  # Matches lines like:  "10.0.100-preview.1.12345"  inside the array.
+  local in_block=false
+  while IFS= read -r line; do
+    if [[ "$line" == *"\"additionalDotNetVersions\""* ]]; then
+      in_block=true
+      continue
+    fi
+    if [[ "$in_block" == true ]]; then
+      if [[ "$line" == *"]"* ]]; then
+        break
+      fi
+      local ver
+      ver=$(echo "$line" | sed -n 's/.*"\([^"]*\)".*/\1/p')
+      if [[ -n "$ver" ]]; then
+        sdk_versions+=("$ver")
+      fi
+    fi
+  done < "$repo_root/global.json"
+
+  local dotnet_root="$repo_root.dotnet"
+
+  echo "Installing SDK(s) '${sdk_versions[*]}' to '$dotnet_root' via dotnetup..."
+
+  local script_dir
+  script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+  local dotnetup_dir="$script_dir/dotnetup"
+  local dotnetup_exe="$dotnetup_dir/dotnetup"
+
+  if ! ShouldUseCachedDotnetup "$dotnetup_exe"; then
+    if ! AcquireDotnetup "$dotnetup_dir"; then
+      Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnetup. Will fall back to standard dotnet-install script."
+      return
+    fi
+  fi
+
+  # Keep dotnetup's manifest under artifacts instead of the user's home dir.
+  export DOTNET_DOTNETUP_DATA_DIR="$artifacts_dir/.dotnetup"
+
+  RunWithoutErrexit "$dotnetup_exe" sdk install "${sdk_versions[@]}" \
+    --install-path "$dotnet_root" \
+    --untracked \
+    --set-default-install false \
+    --interactive false
+  local lastexitcode=$_RunWithoutErrexit
+
+  if [[ $lastexitcode != 0 ]]; then
+    Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install .NET SDK(s) '${sdk_versions[*]}' to '$dotnet_root' using dotnetup (exit code '$lastexitcode'). Will fall back to standard dotnet-install script."
+    return
+  fi
+
+  # Record the installed SDK so CleanOutStage0ToolsetsAndRuntimes does not
+  # later treat this install as stale and wipe it (forcing a build rerun).
+  printf '%s' "$dotnet_sdk_version" > "$dotnet_root/.version"
+}
+
+InstallBootstrapSdkWithDotnetup
+
 # Working around issue https://github.com/dotnet/arcade/issues/7327
-DisableNativeToolsetInstalls=true
+DisableNativeToolsetInstalls=true

eng/dotnetup-shared.ps1

@@ -0,0 +1,128 @@
+# Shared helpers for acquiring dotnetup, dot-sourced by both
+# eng/configure-toolset.ps1 (bootstrap SDK install) and eng/restore-toolset.ps1
+# (test runtime install). This file only defines functions; it has no top-level
+# side effects so it is safe to dot-source multiple times.
+
+# General SDK build helpers (Get-NativeMachineArchitecture, etc.).
+. (Join-Path $PSScriptRoot 'sdk-tools.ps1')
+
+# Returns $true when an already-downloaded dotnetup binary at $DotnetupExe is
+# recent enough (<24h old) and its architecture matches the native machine, so the
+# download can be skipped. Returns $false when dotnetup should be (re)downloaded.
+function Test-ShouldUseCachedDotnetup([string]$DotnetupExe) {
+    if (-not (Test-Path $DotnetupExe)) {
+        return $false
+    }
+
+    # Re-download dotnetup at most once every 24 hours to avoid unnecessary network calls.
+    $age = (Get-Date) - (Get-Item $DotnetupExe).LastWriteTime
+    if ($age.TotalHours -ge 24) {
+        return $false
+    }
+    Write-Host "dotnetup binary is less than 24 hours old; skipping re-download." -ForegroundColor DarkGray
+
+    # dotnetup installs runtimes for its own process architecture, so a cached
+    # binary downloaded under emulation (process arch != native arch) would install
+    # the wrong runtimes. Re-download when the architectures differ.
+    if ((Get-NativeMachineArchitecture) -ne (Get-ProcessMachineArchitecture)) {
+        Write-Host "Native architecture differs from process architecture; re-downloading dotnetup for the native architecture." -ForegroundColor DarkGray
+        return $false
+    }
+
+    return $true
+}
+
+# Runs a PowerShell script in a SEPARATE PowerShell process so that an 'exit'
+# inside it cannot terminate this build host and bypass the caller's try/catch.
+# Uses the current host's own executable to keep pwsh / Windows PowerShell 5.1
+# parity. Throws on non-zero exit code.
+function Invoke-GetDotnetupScript([string]$ScriptPath, [string]$InstallDir, [string]$ErrorLabel) {
+    $psExe = (Get-Process -Id $PID).Path
+    if (-not $psExe) {
+        $psExeName = if ($PSVersionTable.PSEdition -eq 'Core') { 'pwsh' } else { 'powershell' }
+        $psExe = Join-Path $PSHOME $psExeName
+    }
+    if (-not (Test-Path Variable:LASTEXITCODE)) { $global:LASTEXITCODE = 0 }
+
+    # Temporarily set ErrorActionPreference to Continue so that stderr output
+    # from the child process does not become a terminating error (the parent
+    # shell inherits 'Stop' from eng/common/tools.ps1). We rely on
+    # $LASTEXITCODE for error detection instead.
+    $prevEAP = $ErrorActionPreference
+    try {
+        $ErrorActionPreference = 'Continue'
+        & $psExe -NoProfile -ExecutionPolicy Bypass -File $ScriptPath -InstallDir $InstallDir
+    }
+    finally {
+        $ErrorActionPreference = $prevEAP
+    }
+
+    if ($LASTEXITCODE -ne 0) { throw "$ErrorLabel exited with code $LASTEXITCODE." }
+}
+
+# Invokes a native command (e.g. the dotnetup executable)
+# Returns that process exit code WITHOUT letting a non-zero exit become a terminating error.
+# (This covers against $ErrorActionPreference and $PSNativeCommandUseErrorActionPreference)
+function Invoke-DotnetupNativeCommand([scriptblock]$Command) {
+    if (-not (Test-Path Variable:LASTEXITCODE)) { $global:LASTEXITCODE = 0 }
+    $ErrorActionPreference = 'Continue'
+    $PSNativeCommandUseErrorActionPreference = $false
+    try {
+        # Write command output to the host and prevent it from being returned alongside the exit code
+        & $Command | Out-Host
+        return $LASTEXITCODE
+    }
+    catch {
+        Write-Host "dotnetup command failed: $($_.Exception.Message)" -ForegroundColor Yellow
+        if ($LASTEXITCODE -ne 0) { return $LASTEXITCODE }
+        return 1
+    }
+}
+
+# Downloads the public dotnetup installer from aka.ms
+# (https://aka.ms/dotnet/dotnetup/daily/get-dotnetup.ps1) and runs it to install dotnetup into
+# $DotnetupDir. Throws on failure so callers can choose how to react.
+#
+# If a local get-dotnetup.ps1 script exists in the repo (scripts/get-dotnetup.ps1),
+# it is used directly instead of downloading from aka.ms. This supports branches
+# (e.g. release/dnup) that carry the script locally and avoids merge conflicts
+# when code flows between branches with and without the local script.
+function Install-DotnetupFromAkaMs([string]$DotnetupDir) {
+    $repoRoot = (Get-Item $PSScriptRoot).Parent.FullName
+    $localGetter = Join-Path (Join-Path $repoRoot 'scripts') 'get-dotnetup.ps1'
+
+    # Prefer the repo-local script when available (e.g. on release/dnup).
+    if (Test-Path $localGetter) {
+        Write-Host "Using local get-dotnetup.ps1 from '$localGetter'." -ForegroundColor DarkGray
+        Invoke-GetDotnetupScript -ScriptPath $localGetter -InstallDir $DotnetupDir -ErrorLabel "Local get-dotnetup.ps1"
+        return
+    }
+
+    $getterUrl = 'https://aka.ms/dotnet/dotnetup/daily/get-dotnetup.ps1'
+    $getterScript = Join-Path ([System.IO.Path]::GetTempPath()) ("get-dotnetup-{0}.ps1" -f [System.IO.Path]::GetRandomFileName())
+
+    # Download the installer with retry/backoff. Invoke-WebRequest's built-in
+    # -MaximumRetryCount is unavailable on Windows PowerShell 5.1, so retry manually.
+    $maxAttempts = 3
+    for ($attempt = 1; $true; $attempt++) {
+        try {
+            Invoke-WebRequest -Uri $getterUrl -OutFile $getterScript -UseBasicParsing
+            break
+        }
+        catch {
+            if ($attempt -ge $maxAttempts) {
+                throw "Failed to download dotnetup installer from $getterUrl after $maxAttempts attempts: $($_.Exception.Message)"
+            }
+            $delaySeconds = [Math]::Pow(2, $attempt)
+            Write-Host "Download of dotnetup installer failed (attempt $attempt of $maxAttempts): $($_.Exception.Message). Retrying in $delaySeconds seconds..." -ForegroundColor Yellow
+            Start-Sleep -Seconds $delaySeconds
+        }
+    }
+
+    try {
+        Invoke-GetDotnetupScript -ScriptPath $getterScript -InstallDir $DotnetupDir -ErrorLabel "get-dotnetup.ps1"
+    }
+    finally {
+        Remove-Item $getterScript -Force -ErrorAction SilentlyContinue
+    }
+}

eng/dotnetup-shared.sh

@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+
+# Shared helpers for acquiring dotnetup, sourced by both eng/configure-toolset.sh
+# (bootstrap SDK install) and eng/restore-toolset.sh (test runtime install).
+
+# This file only defines functions; it has no top-level side effects so it is safe to source multiple times.
+
+# General SDK build helpers (GetNativeMachineArchitecture, etc.).
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/sdk-tools.sh"
+
+# Returns success (0) when an already-downloaded dotnetup binary at $1 is recent
+# enough (<24h old) and architecturally compatible with the native machine, so the
+# download can be skipped. Returns non-zero when dotnetup should be (re)downloaded.
+function ShouldUseCachedDotnetup {
+  local dotnetup_exe=$1
+  [[ -f "$dotnetup_exe" ]] || return 1
+
+  # Re-download dotnetup at most once every 24 hours to avoid unnecessary network calls.
+  local current_time file_time age_seconds
+  current_time=$(date +%s)
+  file_time=$(stat -c %Y "$dotnetup_exe" 2>/dev/null || stat -f %m "$dotnetup_exe" 2>/dev/null || echo 0)
+  age_seconds=$((current_time - file_time))
+  if [[ $age_seconds -ge 86400 ]]; then
+    return 1
+  fi
+  echo "dotnetup binary is less than 24 hours old; skipping re-download."
+
+  # dotnetup installs runtimes for its own process architecture, so a cached
+  # binary of the wrong architecture (e.g. an x64 dotnetup left on a reused
+  # arm64 agent, or one downloaded under Rosetta 2) would install the wrong
+  # runtimes. Verify the cached binary's actual architecture against the native
+  # architecture and re-download on mismatch rather than trusting uname.
+  if [[ "$(uname)" == "Darwin" ]]; then
+    local native_arch cached_arch=""
+    native_arch="$(GetNativeMachineArchitecture)"
+    if file "$dotnetup_exe" 2>/dev/null | grep -q 'arm64'; then
+      cached_arch="arm64"
+    elif file "$dotnetup_exe" 2>/dev/null | grep -q 'x86_64'; then
+      cached_arch="x64"
+    fi
+    if [[ -n "$cached_arch" && "$cached_arch" != "$native_arch" ]]; then
+      echo "Cached dotnetup architecture ($cached_arch) does not match native architecture ($native_arch); re-downloading."
+      return 1
+    fi
+  fi
+
+  return 0
+}
+
+# Downloads the public dotnetup installer from aka.ms
+# (https://aka.ms/dotnet/dotnetup/daily/get-dotnetup.sh) and runs it to install dotnetup into
+# the directory given by $1. Returns non-zero on failure. Callers run under
+# `set -e`, so invoke via `if ! AcquireDotnetup ...; then` to handle failure.
+#
+# If a local get-dotnetup.sh script exists in the repo (scripts/get-dotnetup.sh),
+# it is used directly instead of downloading from aka.ms. This supports branches
+# (e.g. release/dnup) that carry the script locally and avoids merge conflicts
+# when code flows between branches with and without the local script.
+function AcquireDotnetup {
+  local dotnetup_dir=$1
+  local script_dir
+  script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+  local repo_root
+  repo_root="$(cd "$script_dir/.." && pwd)"
+  local local_getter="$repo_root/scripts/get-dotnetup.sh"
+
+  # Prefer the repo-local script when available (e.g. on release/dnup).
+  if [[ -f "$local_getter" ]]; then
+    echo "Using local get-dotnetup.sh from '$local_getter'."
+    bash "$local_getter" --install-dir "$dotnetup_dir"
+    return $?
+  fi
+
+  local getter_url="https://aka.ms/dotnet/dotnetup/daily/get-dotnetup.sh"
+  local getter_script
+  # Use an explicit template: bare `mktemp` is not portable because BSD/macOS
+  # mktemp requires a template (or -t prefix) and errors without one.
+  getter_script="$(mktemp "${TMPDIR:-/tmp}/get-dotnetup.XXXXXX")"
+
+  local downloaded=false
+  if command -v curl > /dev/null 2>&1; then
+    if curl -fsSL --retry 3 "$getter_url" -o "$getter_script"; then downloaded=true; fi
+  elif command -v wget > /dev/null 2>&1; then
+    if wget -q --tries=3 -O "$getter_script" "$getter_url"; then downloaded=true; fi
+  else
+    echo "Cannot download dotnetup: neither 'curl' nor 'wget' is available on PATH. Install one of them to acquire dotnetup." >&2
+  fi
+
+  local result=0
+  if [[ "$downloaded" != true ]] || ! bash "$getter_script" --install-dir "$dotnetup_dir"; then
+    result=1
+  fi
+
+  rm -f "$getter_script"
+  return $result
+}
+
+# Runs a command with bash 'errexit' (set -e) temporarily disabled so that a non-zero exit code does not abort the calling script
+function RunWithoutErrexit {
+  local restore_errexit=false
+  if [[ $- == *e* ]]; then
+    restore_errexit=true
+    set +e
+  fi
+  "$@"
+  _RunWithoutErrexit=$?
+  if [[ "$restore_errexit" == true ]]; then
+    set -e
+  fi
+  return 0
+}