[10.0.1xx] Use dotnetup preview for SDK acquisition

.gitignore

@@ -40,6 +40,9 @@ cmake/
 # Helix payload
 .dotnet.payload
 
+# dotnetup build output
+eng/dotnetup/
+
 # MSBuild Logs
 **/MSBuild_Logs/MSBuild_pid-*.failure.txt
 

eng/common/templates/vmr-build-pr.yml

@@ -34,7 +34,7 @@ resources:
     type: github
     name: dotnet/dotnet
     endpoint: public
-    ref: refs/heads/main # Set to whatever VMR branch the PR build should insert into
+    ref: refs/heads/release/10.0.1xx
 
 stages:
 - template: /eng/pipelines/templates/stages/vmr-build.yml@vmr

eng/configure-toolset.ps1

@@ -4,3 +4,62 @@
 
 $script:useInstalledDotNetCli = $false
 
+# Shared dotnetup acquisition helpers (architecture detection, cache freshness, download).
+. (Join-Path $PSScriptRoot 'dotnetup-shared.ps1')
+
+# Pre-install the bootstrap SDK pinned in global.json using dotnetup into the
+# repo-local .dotnet directory that arcade's InitializeDotnetCli will pick up.
+#
+# Skipped during VMR / source-build (no network) and when -restore was not requested.
+function InstallBootstrapSdkWithDotnetup() {
+    $dotnetSdkVersion = $GlobalJson.tools.dotnet
+    if ((-not $restore) -or $fromVmr -or [string]::IsNullOrEmpty($dotnetSdkVersion)) {
+        return
+    }
+
+    # Collect all SDK versions to install (primary + any additional).
+    $sdkVersions = @($dotnetSdkVersion)
+    if ($GlobalJson.tools.PSObject.Properties['additionalDotNetVersions']) {
+        $sdkVersions += @($GlobalJson.tools.additionalDotNetVersions | Where-Object { -not [string]::IsNullOrEmpty($_) })
+    }
+
+    $dotnetRoot = Join-Path $RepoRoot '.dotnet'
+
+    Write-Host "Installing SDK(s) '$($sdkVersions -join ', ')' to '$dotnetRoot' via dotnetup..." -ForegroundColor Cyan
+
+    $dotnetupDir = Join-Path $PSScriptRoot 'dotnetup'
+    $dotnetupExe = Join-Path $dotnetupDir (GetExecutableFileName 'dotnetup')
+
+    if (-not (Test-ShouldUseCachedDotnetup $dotnetupExe)) {
+        try {
+            Install-DotnetupFromAkaMs $dotnetupDir
+        }
+        catch {
+            Write-Host "Failed to acquire dotnetup: $($_.Exception.Message). Will fall back to standard dotnet-install script." -ForegroundColor Yellow
+            return
+        }
+    }
+
+    # Keep dotnetup's manifest under artifacts instead of the user's home dir.
+    $env:DOTNET_DOTNETUP_DATA_DIR = Join-Path $ArtifactsDir '.dotnetup'
+
+    if (-not (Test-Path Variable:LASTEXITCODE)) { $global:LASTEXITCODE = 0 }
+    $installExitCode = Invoke-DotnetupNativeCommand {
+        & $dotnetupExe sdk install @sdkVersions `
+            --install-path $dotnetRoot `
+            --untracked `
+            --set-default-install false `
+            --interactive false
+    }
+    if ($installExitCode -ne 0) {
+        Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install .NET SDK(s) '$($sdkVersions -join ', ')' to '$dotnetRoot' using dotnetup (exit code '$installExitCode'). Will fall back to standard dotnet-install script."
+        return
+    }
+
+    # Record the installed SDK so CleanOutStage0ToolsetsAndRuntimes does not
+    # later treat this install as stale and wipe it (forcing a build rerun).
+    Set-Content -Path (Join-Path $dotnetRoot '.version') -Value $dotnetSdkVersion -NoNewline
+}
+
+InstallBootstrapSdkWithDotnetup
+

eng/configure-toolset.sh

@@ -4,5 +4,79 @@
 
 useInstalledDotNetCli="false"
 
+# Shared dotnetup acquisition helpers (architecture detection, cache freshness, download).
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/dotnetup-shared.sh"
+
+# Pre-install the bootstrap SDK pinned in global.json using dotnetup into the
+# repo-local .dotnet directory that arcade's InitializeDotNetCli will pick up.
+#
+# Skipped during VMR / source-build (no network) and when --restore was not requested.
+function InstallBootstrapSdkWithDotnetup {
+  ReadGlobalVersion "dotnet"
+  local dotnet_sdk_version=$_ReadGlobalVersion
+  if [[ "$restore" != true || "$from_vmr" == true || -z "$dotnet_sdk_version" ]]; then
+    return
+  fi
+
+  # Collect all SDK versions to install (primary + any additional).
+  local sdk_versions=("$dotnet_sdk_version")
+  # Extract additionalDotNetVersions from global.json using grep/sed (no python dependency).
+  # Matches lines like:  "10.0.100-preview.1.12345"  inside the array.
+  local in_block=false
+  while IFS= read -r line; do
+    if [[ "$line" == *"\"additionalDotNetVersions\""* ]]; then
+      in_block=true
+      continue
+    fi
+    if [[ "$in_block" == true ]]; then
+      if [[ "$line" == *"]"* ]]; then
+        break
+      fi
+      local ver
+      ver=$(echo "$line" | sed -n 's/.*"\([^"]*\)".*/\1/p')
+      if [[ -n "$ver" ]]; then
+        sdk_versions+=("$ver")
+      fi
+    fi
+  done < "$repo_root/global.json"
+
+  local dotnet_root="$repo_root.dotnet"
+
+  echo "Installing SDK(s) '${sdk_versions[*]}' to '$dotnet_root' via dotnetup..."
+
+  local script_dir
+  script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+  local dotnetup_dir="$script_dir/dotnetup"
+  local dotnetup_exe="$dotnetup_dir/dotnetup"
+
+  if ! ShouldUseCachedDotnetup "$dotnetup_exe"; then
+    if ! AcquireDotnetup "$dotnetup_dir"; then
+      Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnetup. Will fall back to standard dotnet-install script."
+      return
+    fi
+  fi
+
+  # Keep dotnetup's manifest under artifacts instead of the user's home dir.
+  export DOTNET_DOTNETUP_DATA_DIR="$artifacts_dir/.dotnetup"
+
+  RunWithoutErrexit "$dotnetup_exe" sdk install "${sdk_versions[@]}" \
+    --install-path "$dotnet_root" \
+    --untracked \
+    --set-default-install false \
+    --interactive false
+  local lastexitcode=$_RunWithoutErrexit
+
+  if [[ $lastexitcode != 0 ]]; then
+    Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install .NET SDK(s) '${sdk_versions[*]}' to '$dotnet_root' using dotnetup (exit code '$lastexitcode'). Will fall back to standard dotnet-install script."
+    return
+  fi
+
+  # Record the installed SDK so CleanOutStage0ToolsetsAndRuntimes does not
+  # later treat this install as stale and wipe it (forcing a build rerun).
+  printf '%s' "$dotnet_sdk_version" > "$dotnet_root/.version"
+}
+
+InstallBootstrapSdkWithDotnetup
+
 # Working around issue https://github.com/dotnet/arcade/issues/7327
-DisableNativeToolsetInstalls=true
+DisableNativeToolsetInstalls=true

eng/dotnetup-shared.ps1

@@ -0,0 +1,128 @@
+# Shared helpers for acquiring dotnetup, dot-sourced by both
+# eng/configure-toolset.ps1 (bootstrap SDK install) and eng/restore-toolset.ps1
+# (test runtime install). This file only defines functions; it has no top-level
+# side effects so it is safe to dot-source multiple times.
+
+# General SDK build helpers (Get-NativeMachineArchitecture, etc.).
+. (Join-Path $PSScriptRoot 'sdk-tools.ps1')
+
+# Returns $true when an already-downloaded dotnetup binary at $DotnetupExe is
+# recent enough (<24h old) and its architecture matches the native machine, so the
+# download can be skipped. Returns $false when dotnetup should be (re)downloaded.
+function Test-ShouldUseCachedDotnetup([string]$DotnetupExe) {
+    if (-not (Test-Path $DotnetupExe)) {
+        return $false
+    }
+
+    # Re-download dotnetup at most once every 24 hours to avoid unnecessary network calls.
+    $age = (Get-Date) - (Get-Item $DotnetupExe).LastWriteTime
+    if ($age.TotalHours -ge 24) {
+        return $false
+    }
+    Write-Host "dotnetup binary is less than 24 hours old; skipping re-download." -ForegroundColor DarkGray
+
+    # dotnetup installs runtimes for its own process architecture, so a cached
+    # binary downloaded under emulation (process arch != native arch) would install
+    # the wrong runtimes. Re-download when the architectures differ.
+    if ((Get-NativeMachineArchitecture) -ne (Get-ProcessMachineArchitecture)) {
+        Write-Host "Native architecture differs from process architecture; re-downloading dotnetup for the native architecture." -ForegroundColor DarkGray
+        return $false
+    }
+
+    return $true
+}
+
+# Runs a PowerShell script in a SEPARATE PowerShell process so that an 'exit'
+# inside it cannot terminate this build host and bypass the caller's try/catch.
+# Uses the current host's own executable to keep pwsh / Windows PowerShell 5.1
+# parity. Throws on non-zero exit code.
+function Invoke-GetDotnetupScript([string]$ScriptPath, [string]$InstallDir, [string]$ErrorLabel) {
+    $psExe = (Get-Process -Id $PID).Path
+    if (-not $psExe) {
+        $psExeName = if ($PSVersionTable.PSEdition -eq 'Core') { 'pwsh' } else { 'powershell' }
+        $psExe = Join-Path $PSHOME $psExeName
+    }
+    if (-not (Test-Path Variable:LASTEXITCODE)) { $global:LASTEXITCODE = 0 }
+
+    # Temporarily set ErrorActionPreference to Continue so that stderr output
+    # from the child process does not become a terminating error (the parent
+    # shell inherits 'Stop' from eng/common/tools.ps1). We rely on
+    # $LASTEXITCODE for error detection instead.
+    $prevEAP = $ErrorActionPreference
+    try {
+        $ErrorActionPreference = 'Continue'
+        & $psExe -NoProfile -ExecutionPolicy Bypass -File $ScriptPath -InstallDir $InstallDir
+    }
+    finally {
+        $ErrorActionPreference = $prevEAP
+    }
+
+    if ($LASTEXITCODE -ne 0) { throw "$ErrorLabel exited with code $LASTEXITCODE." }
+}
+
+# Invokes a native command (e.g. the dotnetup executable)
+# Returns that process exit code WITHOUT letting a non-zero exit become a terminating error.
+# (This covers against $ErrorActionPreference and $PSNativeCommandUseErrorActionPreference)
+function Invoke-DotnetupNativeCommand([scriptblock]$Command) {
+    if (-not (Test-Path Variable:LASTEXITCODE)) { $global:LASTEXITCODE = 0 }
+    $ErrorActionPreference = 'Continue'
+    $PSNativeCommandUseErrorActionPreference = $false
+    try {
+        # Write command output to the host and prevent it from being returned alongside the exit code
+        & $Command | Out-Host
+        return $LASTEXITCODE
+    }
+    catch {
+        Write-Host "dotnetup command failed: $($_.Exception.Message)" -ForegroundColor Yellow
+        if ($LASTEXITCODE -ne 0) { return $LASTEXITCODE }
+        return 1
+    }
+}
+
+# Downloads the public dotnetup installer from aka.ms
+# (https://aka.ms/dotnet/dotnetup/daily/get-dotnetup.ps1) and runs it to install dotnetup into
+# $DotnetupDir. Throws on failure so callers can choose how to react.
+#
+# If a local get-dotnetup.ps1 script exists in the repo (scripts/get-dotnetup.ps1),
+# it is used directly instead of downloading from aka.ms. This supports branches
+# (e.g. release/dnup) that carry the script locally and avoids merge conflicts
+# when code flows between branches with and without the local script.
+function Install-DotnetupFromAkaMs([string]$DotnetupDir) {
+    $repoRoot = (Get-Item $PSScriptRoot).Parent.FullName
+    $localGetter = Join-Path (Join-Path $repoRoot 'scripts') 'get-dotnetup.ps1'
+
+    # Prefer the repo-local script when available (e.g. on release/dnup).
+    if (Test-Path $localGetter) {
+        Write-Host "Using local get-dotnetup.ps1 from '$localGetter'." -ForegroundColor DarkGray
+        Invoke-GetDotnetupScript -ScriptPath $localGetter -InstallDir $DotnetupDir -ErrorLabel "Local get-dotnetup.ps1"
+        return
+    }
+
+    $getterUrl = 'https://aka.ms/dotnet/dotnetup/daily/get-dotnetup.ps1'
+    $getterScript = Join-Path ([System.IO.Path]::GetTempPath()) ("get-dotnetup-{0}.ps1" -f [System.IO.Path]::GetRandomFileName())
+
+    # Download the installer with retry/backoff. Invoke-WebRequest's built-in
+    # -MaximumRetryCount is unavailable on Windows PowerShell 5.1, so retry manually.
+    $maxAttempts = 3
+    for ($attempt = 1; $true; $attempt++) {
+        try {
+            Invoke-WebRequest -Uri $getterUrl -OutFile $getterScript -UseBasicParsing
+            break
+        }
+        catch {
+            if ($attempt -ge $maxAttempts) {
+                throw "Failed to download dotnetup installer from $getterUrl after $maxAttempts attempts: $($_.Exception.Message)"
+            }
+            $delaySeconds = [Math]::Pow(2, $attempt)
+            Write-Host "Download of dotnetup installer failed (attempt $attempt of $maxAttempts): $($_.Exception.Message). Retrying in $delaySeconds seconds..." -ForegroundColor Yellow
+            Start-Sleep -Seconds $delaySeconds
+        }
+    }
+
+    try {
+        Invoke-GetDotnetupScript -ScriptPath $getterScript -InstallDir $DotnetupDir -ErrorLabel "get-dotnetup.ps1"
+    }
+    finally {
+        Remove-Item $getterScript -Force -ErrorAction SilentlyContinue
+    }
+}