diff --git a/.gitignore b/.gitignore index 3f0661881fe4..81a1c7d9c825 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,9 @@ cmake/ # Helix payload .dotnet.payload +# dotnetup build output +eng/dotnetup/ + # MSBuild Logs **/MSBuild_Logs/MSBuild_pid-*.failure.txt diff --git a/eng/configure-toolset.ps1 b/eng/configure-toolset.ps1 index 9ea9b98010fe..3f4a76739275 100644 --- a/eng/configure-toolset.ps1 +++ b/eng/configure-toolset.ps1 @@ -4,3 +4,62 @@ $script:useInstalledDotNetCli = $false +# Shared dotnetup acquisition helpers (architecture detection, cache freshness, download). +. (Join-Path $PSScriptRoot 'dotnetup-shared.ps1') + +# Pre-install the bootstrap SDK pinned in global.json using dotnetup into the +# repo-local .dotnet directory that arcade's InitializeDotnetCli will pick up. +# +# Skipped during VMR / source-build (no network) and when -restore was not requested. +function InstallBootstrapSdkWithDotnetup() { + $dotnetSdkVersion = $GlobalJson.tools.dotnet + if ((-not $restore) -or $fromVmr -or [string]::IsNullOrEmpty($dotnetSdkVersion)) { + return + } + + # Collect all SDK versions to install (primary + any additional). + $sdkVersions = @($dotnetSdkVersion) + if ($GlobalJson.tools.PSObject.Properties['additionalDotNetVersions']) { + $sdkVersions += @($GlobalJson.tools.additionalDotNetVersions | Where-Object { -not [string]::IsNullOrEmpty($_) }) + } + + $dotnetRoot = Join-Path $RepoRoot '.dotnet' + + Write-Host "Installing SDK(s) '$($sdkVersions -join ', ')' to '$dotnetRoot' via dotnetup..." -ForegroundColor Cyan + + $dotnetupDir = Join-Path $PSScriptRoot 'dotnetup' + $dotnetupExe = Join-Path $dotnetupDir (GetExecutableFileName 'dotnetup') + + if (-not (Test-ShouldUseCachedDotnetup $dotnetupExe)) { + try { + Install-DotnetupFromAkaMs $dotnetupDir + } + catch { + Write-Host "Failed to acquire dotnetup: $($_.Exception.Message). Will fall back to standard dotnet-install script." -ForegroundColor Yellow + return + } + } + + # Keep dotnetup's manifest under artifacts instead of the user's home dir. + $env:DOTNET_DOTNETUP_DATA_DIR = Join-Path $ArtifactsDir '.dotnetup' + + if (-not (Test-Path Variable:LASTEXITCODE)) { $global:LASTEXITCODE = 0 } + $installExitCode = Invoke-DotnetupNativeCommand { + & $dotnetupExe sdk install @sdkVersions ` + --install-path $dotnetRoot ` + --untracked ` + --set-default-install false ` + --interactive false + } + if ($installExitCode -ne 0) { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install .NET SDK(s) '$($sdkVersions -join ', ')' to '$dotnetRoot' using dotnetup (exit code '$installExitCode'). Will fall back to standard dotnet-install script." + return + } + + # Record the installed SDK so CleanOutStage0ToolsetsAndRuntimes does not + # later treat this install as stale and wipe it (forcing a build rerun). + Set-Content -Path (Join-Path $dotnetRoot '.version') -Value $dotnetSdkVersion -NoNewline +} + +InstallBootstrapSdkWithDotnetup + diff --git a/eng/configure-toolset.sh b/eng/configure-toolset.sh index 132c0e8cfa6a..9bea1d0957d3 100644 --- a/eng/configure-toolset.sh +++ b/eng/configure-toolset.sh @@ -4,5 +4,79 @@ useInstalledDotNetCli="false" +# Shared dotnetup acquisition helpers (architecture detection, cache freshness, download). +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/dotnetup-shared.sh" + +# Pre-install the bootstrap SDK pinned in global.json using dotnetup into the +# repo-local .dotnet directory that arcade's InitializeDotNetCli will pick up. +# +# Skipped during VMR / source-build (no network) and when --restore was not requested. +function InstallBootstrapSdkWithDotnetup { + ReadGlobalVersion "dotnet" + local dotnet_sdk_version=$_ReadGlobalVersion + if [[ "$restore" != true || "$from_vmr" == true || -z "$dotnet_sdk_version" ]]; then + return + fi + + # Collect all SDK versions to install (primary + any additional). + local sdk_versions=("$dotnet_sdk_version") + # Extract additionalDotNetVersions from global.json using grep/sed (no python dependency). + # Matches lines like: "10.0.100-preview.1.12345" inside the array. + local in_block=false + while IFS= read -r line; do + if [[ "$line" == *"\"additionalDotNetVersions\""* ]]; then + in_block=true + continue + fi + if [[ "$in_block" == true ]]; then + if [[ "$line" == *"]"* ]]; then + break + fi + local ver + ver=$(echo "$line" | sed -n 's/.*"\([^"]*\)".*/\1/p') + if [[ -n "$ver" ]]; then + sdk_versions+=("$ver") + fi + fi + done < "$repo_root/global.json" + + local dotnet_root="$repo_root.dotnet" + + echo "Installing SDK(s) '${sdk_versions[*]}' to '$dotnet_root' via dotnetup..." + + local script_dir + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + local dotnetup_dir="$script_dir/dotnetup" + local dotnetup_exe="$dotnetup_dir/dotnetup" + + if ! ShouldUseCachedDotnetup "$dotnetup_exe"; then + if ! AcquireDotnetup "$dotnetup_dir"; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnetup. Will fall back to standard dotnet-install script." + return + fi + fi + + # Keep dotnetup's manifest under artifacts instead of the user's home dir. + export DOTNET_DOTNETUP_DATA_DIR="$artifacts_dir/.dotnetup" + + RunWithoutErrexit "$dotnetup_exe" sdk install "${sdk_versions[@]}" \ + --install-path "$dotnet_root" \ + --untracked \ + --set-default-install false \ + --interactive false + local lastexitcode=$_RunWithoutErrexit + + if [[ $lastexitcode != 0 ]]; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install .NET SDK(s) '${sdk_versions[*]}' to '$dotnet_root' using dotnetup (exit code '$lastexitcode'). Will fall back to standard dotnet-install script." + return + fi + + # Record the installed SDK so CleanOutStage0ToolsetsAndRuntimes does not + # later treat this install as stale and wipe it (forcing a build rerun). + printf '%s' "$dotnet_sdk_version" > "$dotnet_root/.version" +} + +InstallBootstrapSdkWithDotnetup + # Working around issue https://github.com/dotnet/arcade/issues/7327 -DisableNativeToolsetInstalls=true \ No newline at end of file +DisableNativeToolsetInstalls=true diff --git a/eng/dotnetup-shared.ps1 b/eng/dotnetup-shared.ps1 new file mode 100644 index 000000000000..c50280104412 --- /dev/null +++ b/eng/dotnetup-shared.ps1 @@ -0,0 +1,128 @@ +# Shared helpers for acquiring dotnetup, dot-sourced by both +# eng/configure-toolset.ps1 (bootstrap SDK install) and eng/restore-toolset.ps1 +# (test runtime install). This file only defines functions; it has no top-level +# side effects so it is safe to dot-source multiple times. + +# General SDK build helpers (Get-NativeMachineArchitecture, etc.). +. (Join-Path $PSScriptRoot 'sdk-tools.ps1') + +# Returns $true when an already-downloaded dotnetup binary at $DotnetupExe is +# recent enough (<24h old) and its architecture matches the native machine, so the +# download can be skipped. Returns $false when dotnetup should be (re)downloaded. +function Test-ShouldUseCachedDotnetup([string]$DotnetupExe) { + if (-not (Test-Path $DotnetupExe)) { + return $false + } + + # Re-download dotnetup at most once every 24 hours to avoid unnecessary network calls. + $age = (Get-Date) - (Get-Item $DotnetupExe).LastWriteTime + if ($age.TotalHours -ge 24) { + return $false + } + Write-Host "dotnetup binary is less than 24 hours old; skipping re-download." -ForegroundColor DarkGray + + # dotnetup installs runtimes for its own process architecture, so a cached + # binary downloaded under emulation (process arch != native arch) would install + # the wrong runtimes. Re-download when the architectures differ. + if ((Get-NativeMachineArchitecture) -ne (Get-ProcessMachineArchitecture)) { + Write-Host "Native architecture differs from process architecture; re-downloading dotnetup for the native architecture." -ForegroundColor DarkGray + return $false + } + + return $true +} + +# Runs a PowerShell script in a SEPARATE PowerShell process so that an 'exit' +# inside it cannot terminate this build host and bypass the caller's try/catch. +# Uses the current host's own executable to keep pwsh / Windows PowerShell 5.1 +# parity. Throws on non-zero exit code. +function Invoke-GetDotnetupScript([string]$ScriptPath, [string]$InstallDir, [string]$ErrorLabel) { + $psExe = (Get-Process -Id $PID).Path + if (-not $psExe) { + $psExeName = if ($PSVersionTable.PSEdition -eq 'Core') { 'pwsh' } else { 'powershell' } + $psExe = Join-Path $PSHOME $psExeName + } + if (-not (Test-Path Variable:LASTEXITCODE)) { $global:LASTEXITCODE = 0 } + + # Temporarily set ErrorActionPreference to Continue so that stderr output + # from the child process does not become a terminating error (the parent + # shell inherits 'Stop' from eng/common/tools.ps1). We rely on + # $LASTEXITCODE for error detection instead. + $prevEAP = $ErrorActionPreference + try { + $ErrorActionPreference = 'Continue' + & $psExe -NoProfile -ExecutionPolicy Bypass -File $ScriptPath -InstallDir $InstallDir + } + finally { + $ErrorActionPreference = $prevEAP + } + + if ($LASTEXITCODE -ne 0) { throw "$ErrorLabel exited with code $LASTEXITCODE." } +} + +# Invokes a native command (e.g. the dotnetup executable) +# Returns that process exit code WITHOUT letting a non-zero exit become a terminating error. +# (This covers against $ErrorActionPreference and $PSNativeCommandUseErrorActionPreference) +function Invoke-DotnetupNativeCommand([scriptblock]$Command) { + if (-not (Test-Path Variable:LASTEXITCODE)) { $global:LASTEXITCODE = 0 } + $ErrorActionPreference = 'Continue' + $PSNativeCommandUseErrorActionPreference = $false + try { + # Write command output to the host and prevent it from being returned alongside the exit code + & $Command | Out-Host + return $LASTEXITCODE + } + catch { + Write-Host "dotnetup command failed: $($_.Exception.Message)" -ForegroundColor Yellow + if ($LASTEXITCODE -ne 0) { return $LASTEXITCODE } + return 1 + } +} + +# Downloads the public dotnetup installer from aka.ms +# (https://aka.ms/dotnet/dotnetup/daily/get-dotnetup.ps1) and runs it to install dotnetup into +# $DotnetupDir. Throws on failure so callers can choose how to react. +# +# If a local get-dotnetup.ps1 script exists in the repo (scripts/get-dotnetup.ps1), +# it is used directly instead of downloading from aka.ms. This supports branches +# (e.g. release/dnup) that carry the script locally and avoids merge conflicts +# when code flows between branches with and without the local script. +function Install-DotnetupFromAkaMs([string]$DotnetupDir) { + $repoRoot = (Get-Item $PSScriptRoot).Parent.FullName + $localGetter = Join-Path (Join-Path $repoRoot 'scripts') 'get-dotnetup.ps1' + + # Prefer the repo-local script when available (e.g. on release/dnup). + if (Test-Path $localGetter) { + Write-Host "Using local get-dotnetup.ps1 from '$localGetter'." -ForegroundColor DarkGray + Invoke-GetDotnetupScript -ScriptPath $localGetter -InstallDir $DotnetupDir -ErrorLabel "Local get-dotnetup.ps1" + return + } + + $getterUrl = 'https://aka.ms/dotnet/dotnetup/daily/get-dotnetup.ps1' + $getterScript = Join-Path ([System.IO.Path]::GetTempPath()) ("get-dotnetup-{0}.ps1" -f [System.IO.Path]::GetRandomFileName()) + + # Download the installer with retry/backoff. Invoke-WebRequest's built-in + # -MaximumRetryCount is unavailable on Windows PowerShell 5.1, so retry manually. + $maxAttempts = 3 + for ($attempt = 1; $true; $attempt++) { + try { + Invoke-WebRequest -Uri $getterUrl -OutFile $getterScript -UseBasicParsing + break + } + catch { + if ($attempt -ge $maxAttempts) { + throw "Failed to download dotnetup installer from $getterUrl after $maxAttempts attempts: $($_.Exception.Message)" + } + $delaySeconds = [Math]::Pow(2, $attempt) + Write-Host "Download of dotnetup installer failed (attempt $attempt of $maxAttempts): $($_.Exception.Message). Retrying in $delaySeconds seconds..." -ForegroundColor Yellow + Start-Sleep -Seconds $delaySeconds + } + } + + try { + Invoke-GetDotnetupScript -ScriptPath $getterScript -InstallDir $DotnetupDir -ErrorLabel "get-dotnetup.ps1" + } + finally { + Remove-Item $getterScript -Force -ErrorAction SilentlyContinue + } +} diff --git a/eng/dotnetup-shared.sh b/eng/dotnetup-shared.sh new file mode 100644 index 000000000000..8d4b188101fa --- /dev/null +++ b/eng/dotnetup-shared.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +# Shared helpers for acquiring dotnetup, sourced by both eng/configure-toolset.sh +# (bootstrap SDK install) and eng/restore-toolset.sh (test runtime install). + +# This file only defines functions; it has no top-level side effects so it is safe to source multiple times. + +# General SDK build helpers (GetNativeMachineArchitecture, etc.). +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/sdk-tools.sh" + +# Returns success (0) when an already-downloaded dotnetup binary at $1 is recent +# enough (<24h old) and architecturally compatible with the native machine, so the +# download can be skipped. Returns non-zero when dotnetup should be (re)downloaded. +function ShouldUseCachedDotnetup { + local dotnetup_exe=$1 + [[ -f "$dotnetup_exe" ]] || return 1 + + # Re-download dotnetup at most once every 24 hours to avoid unnecessary network calls. + local current_time file_time age_seconds + current_time=$(date +%s) + file_time=$(stat -c %Y "$dotnetup_exe" 2>/dev/null || stat -f %m "$dotnetup_exe" 2>/dev/null || echo 0) + age_seconds=$((current_time - file_time)) + if [[ $age_seconds -ge 86400 ]]; then + return 1 + fi + echo "dotnetup binary is less than 24 hours old; skipping re-download." + + # dotnetup installs runtimes for its own process architecture, so a cached + # binary of the wrong architecture (e.g. an x64 dotnetup left on a reused + # arm64 agent, or one downloaded under Rosetta 2) would install the wrong + # runtimes. Verify the cached binary's actual architecture against the native + # architecture and re-download on mismatch rather than trusting uname. + if [[ "$(uname)" == "Darwin" ]]; then + local native_arch cached_arch="" + native_arch="$(GetNativeMachineArchitecture)" + if file "$dotnetup_exe" 2>/dev/null | grep -q 'arm64'; then + cached_arch="arm64" + elif file "$dotnetup_exe" 2>/dev/null | grep -q 'x86_64'; then + cached_arch="x64" + fi + if [[ -n "$cached_arch" && "$cached_arch" != "$native_arch" ]]; then + echo "Cached dotnetup architecture ($cached_arch) does not match native architecture ($native_arch); re-downloading." + return 1 + fi + fi + + return 0 +} + +# Downloads the public dotnetup installer from aka.ms +# (https://aka.ms/dotnet/dotnetup/daily/get-dotnetup.sh) and runs it to install dotnetup into +# the directory given by $1. Returns non-zero on failure. Callers run under +# `set -e`, so invoke via `if ! AcquireDotnetup ...; then` to handle failure. +# +# If a local get-dotnetup.sh script exists in the repo (scripts/get-dotnetup.sh), +# it is used directly instead of downloading from aka.ms. This supports branches +# (e.g. release/dnup) that carry the script locally and avoids merge conflicts +# when code flows between branches with and without the local script. +function AcquireDotnetup { + local dotnetup_dir=$1 + local script_dir + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + local repo_root + repo_root="$(cd "$script_dir/.." && pwd)" + local local_getter="$repo_root/scripts/get-dotnetup.sh" + + # Prefer the repo-local script when available (e.g. on release/dnup). + if [[ -f "$local_getter" ]]; then + echo "Using local get-dotnetup.sh from '$local_getter'." + bash "$local_getter" --install-dir "$dotnetup_dir" + return $? + fi + + local getter_url="https://aka.ms/dotnet/dotnetup/daily/get-dotnetup.sh" + local getter_script + # Use an explicit template: bare `mktemp` is not portable because BSD/macOS + # mktemp requires a template (or -t prefix) and errors without one. + getter_script="$(mktemp "${TMPDIR:-/tmp}/get-dotnetup.XXXXXX")" + + local downloaded=false + if command -v curl > /dev/null 2>&1; then + if curl -fsSL --retry 3 "$getter_url" -o "$getter_script"; then downloaded=true; fi + elif command -v wget > /dev/null 2>&1; then + if wget -q --tries=3 -O "$getter_script" "$getter_url"; then downloaded=true; fi + else + echo "Cannot download dotnetup: neither 'curl' nor 'wget' is available on PATH. Install one of them to acquire dotnetup." >&2 + fi + + local result=0 + if [[ "$downloaded" != true ]] || ! bash "$getter_script" --install-dir "$dotnetup_dir"; then + result=1 + fi + + rm -f "$getter_script" + return $result +} + +# Runs a command with bash 'errexit' (set -e) temporarily disabled so that a non-zero exit code does not abort the calling script +function RunWithoutErrexit { + local restore_errexit=false + if [[ $- == *e* ]]; then + restore_errexit=true + set +e + fi + "$@" + _RunWithoutErrexit=$? + if [[ "$restore_errexit" == true ]]; then + set -e + fi + return 0 +} diff --git a/eng/restore-toolset.ps1 b/eng/restore-toolset.ps1 index 5b3fb632f218..8fb9f53a868f 100644 --- a/eng/restore-toolset.ps1 +++ b/eng/restore-toolset.ps1 @@ -1,48 +1,79 @@ +# Shared dotnetup acquisition helpers (architecture detection, cache freshness, download). +. (Join-Path $PSScriptRoot 'dotnetup-shared.ps1') + + function InitializeCustomSDKToolset { - if ($env:TestFullMSBuild -eq "true") { - $env:DOTNET_SDK_TEST_MSBUILD_PATH = InitializeVisualStudioMSBuild -install:$true -vsRequirements:$GlobalJson.tools.'vs-opt' - Write-Host "INFO: Tests will run against full MSBuild in $env:DOTNET_SDK_TEST_MSBUILD_PATH" - } - - if (-not $restore) { - return - } - - # The following frameworks and tools are used only for testing. - # Do not attempt to install them when building in the VMR. - if ($fromVmr) { - return - } - - $cli = InitializeDotnetCli -install:$true - InstallDotNetSharedFramework "6.0.0" - InstallDotNetSharedFramework "7.0.0" - InstallDotNetSharedFramework "8.0.0" - InstallDotNetSharedFramework "9.0.0" - - CreateBuildEnvScripts - CreateVSShortcut - InstallNuget + if ($env:TestFullMSBuild -eq "true") { + $env:DOTNET_SDK_TEST_MSBUILD_PATH = InitializeVisualStudioMSBuild -install:$true -vsRequirements:$GlobalJson.tools.'vs-opt' + Write-Host "INFO: Tests will run against full MSBuild in $env:DOTNET_SDK_TEST_MSBUILD_PATH" + } + + if (-not $restore) { + return + } + + # The following frameworks and tools are used only for testing. + # Do not attempt to install them when building in the VMR. + if ($fromVmr) { + return + } + + $cli = InitializeDotnetCli -install:$true + + # Redirect dotnetup data directory under artifacts so build scripts + # don't read/write the user's home-folder manifest. + $env:DOTNET_DOTNETUP_DATA_DIR = Join-Path $ArtifactsDir ".dotnetup" + + # The following shared frameworks are only needed for testing. + # Set DOTNET_INSTALL_TEST_RUNTIMES=false to skip (e.g. cross-build containers with limited disk). + if ($env:DOTNET_INSTALL_TEST_RUNTIMES -ne 'false') { + $runtimeSpecs = @("6.0", "7.0", "8.0", "9.0", "10.0") + # Also install the exact runtime versions that arcade's toolset requires + # (from Version.Details.props) so tests can target those specific versions. + $runtimeSpecs += Get-CurrentRuntimeToolsetSpecs + + $nativeArch = Get-NativeMachineArchitecture + if ((-not [string]::IsNullOrEmpty($env:TARGET_ARCHITECTURE)) -and ($env:TARGET_ARCHITECTURE -ne $nativeArch)) { + # Cross-build (e.g. an x64 host producing an arm64 test payload). The host cannot execute + # target-architecture runtimes, so installing them into the host .dotnet would break host + # tools that must load a shared framework there (e.g. the NuGet credential provider, whose + # libhostpolicy load fails on an architecture mismatch). Instead, download the + # target-architecture test runtimes into a sidecar folder under artifacts. The matching + # OverlayCrossArchTestRuntimes target in src/Layout/redist/targets/OverlaySdkOnLKG.targets + # copies these into the test host that ships to Helix, where they run on + # target-architecture hardware. The host .dotnet keeps only host-architecture runtimes; + # host tools roll forward to the host SDK runtime. + $sidecarDir = Join-Path (Join-Path $ArtifactsDir "test-runtimes") $env:TARGET_ARCHITECTURE + Write-Host "Cross-build detected (host '$nativeArch', target '$env:TARGET_ARCHITECTURE'). Installing target-architecture test runtimes into sidecar '$sidecarDir' for the Helix test payload." + New-Item -ItemType Directory -Force -Path $sidecarDir | Out-Null + InstallDotNetSharedFrameworks -RuntimeSpecs $runtimeSpecs -DotNetRoot $sidecarDir -Architecture $env:TARGET_ARCHITECTURE + } + else { + InstallDotNetSharedFrameworks -RuntimeSpecs $runtimeSpecs -DotNetRoot $env:DOTNET_INSTALL_DIR + } + } + + CreateBuildEnvScripts + CreateVSShortcut + InstallNuget } function InstallNuGet { - $NugetInstallDir = Join-Path $ArtifactsDir ".nuget" - $NugetExe = Join-Path $NugetInstallDir "nuget.exe" + $NugetInstallDir = Join-Path $ArtifactsDir ".nuget" + $NugetExe = Join-Path $NugetInstallDir "nuget.exe" - if (!(Test-Path -Path $NugetExe)) { - Create-Directory $NugetInstallDir - Invoke-WebRequest "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe" -UseBasicParsing -OutFile $NugetExe - } + if (!(Test-Path -Path $NugetExe)) { + Create-Directory $NugetInstallDir + Invoke-WebRequest "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe" -UseBasicParsing -OutFile $NugetExe + } } -function CreateBuildEnvScripts() -{ - Create-Directory $ArtifactsDir - $scriptPath = Join-Path $ArtifactsDir "sdk-build-env.bat" - $scriptContents = @" +function CreateBuildEnvScripts() { + Create-Directory $ArtifactsDir + $scriptPath = Join-Path $ArtifactsDir "sdk-build-env.bat" + $scriptContents = @" @echo off title SDK Build ($RepoRoot) -set DOTNET_MULTILEVEL_LOOKUP=0 REM https://aka.ms/vs/unsigned-dotnet-debugger-lib set VSDebugger_ValidateDotnetDebugLibSignatures=0 @@ -56,13 +87,12 @@ set DOTNET_ADD_GLOBAL_TOOLS_TO_PATH=0 DOSKEY killdotnet=taskkill /F /IM dotnet.exe /T ^& taskkill /F /IM VSTest.Console.exe /T ^& taskkill /F /IM msbuild.exe /T "@ - Out-File -FilePath $scriptPath -InputObject $scriptContents -Encoding ASCII + Out-File -FilePath $scriptPath -InputObject $scriptContents -Encoding ASCII - Create-Directory $ArtifactsDir - $scriptPath = Join-Path $ArtifactsDir "sdk-build-env.ps1" - $scriptContents = @" + Create-Directory $ArtifactsDir + $scriptPath = Join-Path $ArtifactsDir "sdk-build-env.ps1" + $scriptContents = @" `$host.ui.RawUI.WindowTitle = "SDK Build ($RepoRoot)" -`$env:DOTNET_MULTILEVEL_LOOKUP=0 # https://aka.ms/vs/unsigned-dotnet-debugger-lib `$env:VSDebugger_ValidateDotnetDebugLibSignatures=0 @@ -80,89 +110,215 @@ function killdotnet { } "@ - Out-File -FilePath $scriptPath -InputObject $scriptContents -Encoding ASCII + Out-File -FilePath $scriptPath -InputObject $scriptContents -Encoding ASCII +} + +function CreateVSShortcut() { + # https://github.com/microsoft/vswhere/wiki/Installing + $installerPath = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer" + if (-Not (Test-Path -Path $installerPath)) { + return + } + + $versionFilePath = Join-Path $RepoRoot 'src\Layout\redist\minimumMSBuildVersion' + # Gets the first digit (ex. 17) and appends '.0' to it. + $vsMajorVersion = "$(((Get-Content $versionFilePath).Split('.'))[0]).0" + $devenvPath = (& "$installerPath\vswhere.exe" -all -prerelease -latest -version $vsMajorVersion -find Common7\IDE\devenv.exe) | Select-Object -First 1 + if (-Not $devenvPath) { + return + } + + $scriptPath = Join-Path $ArtifactsDir 'sdk-build-env.ps1' + $slnPath = Join-Path $RepoRoot 'sdk.slnx' + $commandToLaunch = "& '$scriptPath'; & '$devenvPath' '$slnPath'" + $powershellPath = '%SystemRoot%\system32\WindowsPowerShell\v1.0\powershell.exe' + $shortcutPath = Join-Path $ArtifactsDir 'VS with sdk.slnx.lnk' + + # https://stackoverflow.com/a/9701907/294804 + # https://learn.microsoft.com/en-us/troubleshoot/windows-client/admin-development/create-desktop-shortcut-with-wsh + $wsShell = New-Object -ComObject WScript.Shell + $shortcut = $wsShell.CreateShortcut($shortcutPath) + $shortcut.TargetPath = $powershellPath + $shortcut.Arguments = "-WindowStyle Hidden -ExecutionPolicy Bypass -Command ""$commandToLaunch""" + $shortcut.IconLocation = $devenvPath + $shortcut.WindowStyle = 7 # Minimized + $shortcut.Save() } -function CreateVSShortcut() -{ - # https://github.com/microsoft/vswhere/wiki/Installing - $installerPath = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer" - if(-Not (Test-Path -Path $installerPath)) - { - return - } - - $versionFilePath = Join-Path $RepoRoot 'src\Layout\redist\minimumMSBuildVersion' - # Gets the first digit (ex. 17) and appends '.0' to it. - $vsMajorVersion = "$(((Get-Content $versionFilePath).Split('.'))[0]).0" - $devenvPath = (& "$installerPath\vswhere.exe" -all -prerelease -latest -version $vsMajorVersion -find Common7\IDE\devenv.exe) | Select-Object -First 1 - if(-Not $devenvPath) - { - return - } - - $scriptPath = Join-Path $ArtifactsDir 'sdk-build-env.ps1' - $slnPath = Join-Path $RepoRoot 'sdk.slnx' - $commandToLaunch = "& '$scriptPath'; & '$devenvPath' '$slnPath'" - $powershellPath = '%SystemRoot%\system32\WindowsPowerShell\v1.0\powershell.exe' - $shortcutPath = Join-Path $ArtifactsDir 'VS with sdk.slnx.lnk' - - # https://stackoverflow.com/a/9701907/294804 - # https://learn.microsoft.com/en-us/troubleshoot/windows-client/admin-development/create-desktop-shortcut-with-wsh - $wsShell = New-Object -ComObject WScript.Shell - $shortcut = $wsShell.CreateShortcut($shortcutPath) - $shortcut.TargetPath = $powershellPath - $shortcut.Arguments = "-WindowStyle Hidden -ExecutionPolicy Bypass -Command ""$commandToLaunch""" - $shortcut.IconLocation = $devenvPath - $shortcut.WindowStyle = 7 # Minimized - $shortcut.Save() +# Maps a dotnetup channel version (e.g. "9.0") to the specific version +# expected by the dotnet-install script's -Version parameter (e.g. "9.0.0"). +# Full versions (e.g. "9.0.0-preview.5.24306.7") are passed through unchanged. +function ConvertTo-DotNetInstallScriptVersion([string]$version) { + if ($version -match '^\d+\.\d+$') { + return "$version.0" + } + + return $version } -function InstallDotNetSharedFramework([string]$version) { - $dotnetRoot = $env:DOTNET_INSTALL_DIR - $fxDir = Join-Path $dotnetRoot "shared\Microsoft.NETCore.App\$version" +function Get-VersionDetailsProperty([string]$propertyName) { + $versionDetails = [xml](Get-Content -Raw -Path (Join-Path $RepoRoot 'eng\Version.Details.props')) + $property = $versionDetails.SelectSingleNode("//$propertyName") + if ($null -eq $property) { + return "" + } + + return $property.InnerText +} - if (!(Test-Path $fxDir)) { - $installScript = GetDotNetInstallScript $dotnetRoot - & $installScript -Version $version -InstallDir $dotnetRoot -Runtime "dotnet" -SkipNonVersionedFiles +function Get-CurrentRuntimeToolsetSpecs() { + $runtimeVersion = Get-VersionDetailsProperty 'MicrosoftNETCoreAppRefPackageVersion' + $aspNetCoreVersion = Get-VersionDetailsProperty 'MicrosoftAspNetCoreAppRefPackageVersion' - if($lastExitCode -ne 0) { - throw "Failed to install shared Framework $version to '$dotnetRoot' (exit code '$lastExitCode')." + $specs = @() + if (-not [string]::IsNullOrEmpty($runtimeVersion)) { + $specs += $runtimeVersion + } + if (-not [string]::IsNullOrEmpty($aspNetCoreVersion)) { + $specs += "aspnetcore@$aspNetCoreVersion" + } + + return $specs +} + +# Maps a dotnetup component (e.g. 'aspnetcore', 'windowsdesktop' or 'dotnet') +# to the name of its shared-framework folder under \shared. +function Get-SharedFrameworkName([string]$component) { + switch ($component) { + 'aspnetcore' { return 'Microsoft.AspNetCore.App' } + 'windowsdesktop' { return 'Microsoft.WindowsDesktop.App' } + default { return 'Microsoft.NETCore.App' } + } +} + +# Returns the shared-framework directory for a component +# (e.g. \shared\Microsoft.AspNetCore.App). +function Get-SharedFrameworkPath([string]$dotNetRoot, [string]$component) { + return Join-Path $dotNetRoot "shared\$(Get-SharedFrameworkName $component)" +} + +# Tests whether a shared framework matching $version (a major.minor channel +# such as 6.0 or an exact version) is already present on disk for $component. +function Test-SharedFrameworkInstalled([string]$dotNetRoot, [string]$component, [string]$version) { + $fxRoot = Get-SharedFrameworkPath $dotNetRoot $component + + # Only a major.minor channel (e.g. 6.0) should match any patch via a wildcard. + # An exact version must match an exact folder so that, for example, 8.0.1 does + # not spuriously match an installed 8.0.10. + if ($version -match '^\d+\.\d+$') { + return [bool](Test-Path -PathType Container (Join-Path $fxRoot "$version*")) + } + + return [bool](Test-Path -PathType Container (Join-Path $fxRoot $version)) +} + +function InstallDotNetSharedFrameworks([string[]]$runtimeSpecs, [string]$dotNetRoot, [string]$architecture = "") { + # Skip if every requested framework is already on disk. Accept either a + # dotnet runtime version/channel or a component@version spec such as + # aspnetcore@11.0.0-preview.6. Treat major.minor channels as present if any + # matching patch (e.g. 6.0.36) exists. + $runtimeSpecsToInstall = @($runtimeSpecs | Where-Object { + $component, $version = if ($_ -match '^([^@]+)@(.+)$') { $matches[1], $matches[2] } else { 'dotnet', $_ } + -not (Test-SharedFrameworkInstalled $dotnetRoot $component $version) + }) + if ($runtimeSpecsToInstall.Count -eq 0) { + return + } + + # dotnetup installs runtimes for its own process architecture and has no + # architecture override (InstallerUtilities.GetDefaultInstallArchitecture uses + # RuntimeInformation.ProcessArchitecture). On a cross-build (e.g. an x64 host + # producing an arm64 test payload), dotnetup would silently install the host + # architecture, so the test runtimes would not match the target Helix queue. + # When a specific architecture is requested, use the dotnet-install script + # directly since it honors -Architecture. + if (-not [string]::IsNullOrEmpty($architecture)) { + InstallDotNetSharedFrameworksWithInstallScript -RuntimeSpecs $runtimeSpecsToInstall -DotNetRoot $dotnetRoot -Architecture $architecture + return + } + + $dotnetupDir = Join-Path $PSScriptRoot "dotnetup" + $dotnetupExe = Join-Path $dotnetupDir (GetExecutableFileName "dotnetup") + + if (-not (Test-ShouldUseCachedDotnetup $dotnetupExe)) { + try { + Install-DotnetupFromAkaMs $dotnetupDir + } + catch { + Write-Host "Failed to acquire dotnetup ($($_.Exception.Message)); falling back to dotnet install script." -ForegroundColor Yellow + InstallDotNetSharedFrameworksWithInstallScript -RuntimeSpecs $runtimeSpecsToInstall -DotNetRoot $dotnetRoot -Architecture $architecture + return + } + } + + if (-not (Test-Path Variable:LASTEXITCODE)) { $global:LASTEXITCODE = 0 } + $installExitCode = Invoke-DotnetupNativeCommand { + & $dotnetupExe runtime install @runtimeSpecsToInstall --install-path $dotnetRoot --set-default-install false --untracked --interactive false + } + + if ($installExitCode -ne 0) { + Write-Host "Failed to install shared frameworks ($($runtimeSpecsToInstall -join ', ')) to '$dotnetRoot' using dotnetup (exit code '$installExitCode'); falling back to dotnet install script." -ForegroundColor Yellow + InstallDotNetSharedFrameworksWithInstallScript -RuntimeSpecs $runtimeSpecsToInstall -DotNetRoot $dotnetRoot -Architecture $architecture + } +} + +function InstallDotNetSharedFrameworksWithInstallScript([string[]]$runtimeSpecs, [string]$dotNetRoot, [string]$architecture = "") { + $installScript = GetDotNetInstallScript $dotNetRoot + foreach ($spec in $runtimeSpecs) { + $component, $version = if ($spec -match '^([^@]+)@(.+)$') { $matches[1], $matches[2] } else { 'dotnet', $spec } + $installVersion = ConvertTo-DotNetInstallScriptVersion $version + $installArgs = @{ + Version = $installVersion + InstallDir = $dotNetRoot + Runtime = $component + SkipNonVersionedFiles = $true + } + if (-not [string]::IsNullOrEmpty($architecture)) { + $installArgs.Architecture = $architecture + } + + $global:LASTEXITCODE = 0 + & $installScript @installArgs + $installScriptExitCode = $LASTEXITCODE + + $frameworkInstalled = Test-SharedFrameworkInstalled $dotNetRoot $component $version + + if ($installScriptExitCode -ne 0 -or -not $frameworkInstalled) { + $architectureMessage = if ([string]::IsNullOrEmpty($architecture)) { "" } else { " for architecture '$architecture'" } + throw "Failed to install shared framework $version to '$dotNetRoot' using dotnet install script$architectureMessage (exit code '$installScriptExitCode', installed '$frameworkInstalled')." + } } - } } # Let's clear out the stage-zero folders that map to the current runtime to keep stage 2 clean function CleanOutStage0ToolsetsAndRuntimes { - $GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot 'global.json') | ConvertFrom-Json - $dotnetSdkVersion = $GlobalJson.tools.dotnet - $dotnetRoot = $env:DOTNET_INSTALL_DIR - $versionPath = Join-Path $dotnetRoot '.version' - $aspnetRuntimePath = [IO.Path]::Combine( $dotnetRoot, 'shared' ,'Microsoft.AspNetCore.App') - $coreRuntimePath = [IO.Path]::Combine( $dotnetRoot, 'shared' ,'Microsoft.NETCore.App') - $wdRuntimePath = [IO.Path]::Combine( $dotnetRoot, 'shared', 'Microsoft.WindowsDesktop.App') - $sdkPath = Join-Path $dotnetRoot 'sdk' - $majorVersion = $dotnetSdkVersion.Substring(0,1) - - if (Test-Path($versionPath)) { - $lastInstalledSDK = Get-Content -Raw -Path ($versionPath) - if ($lastInstalledSDK -ne $dotnetSdkVersion) - { - $dotnetSdkVersion | Out-File -FilePath $versionPath -NoNewline - Remove-Item (Join-Path $aspnetRuntimePath "$majorVersion.*") -Recurse - Remove-Item (Join-Path $coreRuntimePath "$majorVersion.*") -Recurse - Remove-Item (Join-Path $wdRuntimePath "$majorVersion.*") -Recurse - Remove-Item (Join-Path $sdkPath "*") -Recurse - Remove-Item (Join-Path $dotnetRoot "packs") -Recurse - Remove-Item (Join-Path $dotnetRoot "sdk-manifests") -Recurse - Remove-Item (Join-Path $dotnetRoot "templates") -Recurse - throw "Installed a new SDK, deleting existing shared frameworks and sdk folders. Please rerun build" - } - } - else - { - $dotnetSdkVersion | Out-File -FilePath $versionPath -NoNewline - } + $GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot 'global.json') | ConvertFrom-Json + $dotnetSdkVersion = $GlobalJson.tools.dotnet + $dotnetRoot = $env:DOTNET_INSTALL_DIR + $versionPath = Join-Path $dotnetRoot '.version' + $aspnetRuntimePath = Get-SharedFrameworkPath $dotnetRoot 'aspnetcore' + $coreRuntimePath = Get-SharedFrameworkPath $dotnetRoot 'dotnet' + $wdRuntimePath = Get-SharedFrameworkPath $dotnetRoot 'windowsdesktop' + $sdkPath = Join-Path $dotnetRoot 'sdk' + $majorVersion = $dotnetSdkVersion.Split('.')[0] + + if (Test-Path($versionPath)) { + $lastInstalledSDK = Get-Content -Raw -Path ($versionPath) + if ($lastInstalledSDK -ne $dotnetSdkVersion) { + $dotnetSdkVersion | Out-File -FilePath $versionPath -NoNewline + Remove-Item (Join-Path $aspnetRuntimePath "$majorVersion.*") -Recurse + Remove-Item (Join-Path $coreRuntimePath "$majorVersion.*") -Recurse + Remove-Item (Join-Path $wdRuntimePath "$majorVersion.*") -Recurse + Remove-Item (Join-Path $sdkPath "*") -Recurse + Remove-Item (Join-Path $dotnetRoot "packs") -Recurse + Remove-Item (Join-Path $dotnetRoot "sdk-manifests") -Recurse + Remove-Item (Join-Path $dotnetRoot "templates") -Recurse + throw "Installed a new SDK, deleting existing shared frameworks and sdk folders. Please rerun build" + } + } + else { + $dotnetSdkVersion | Out-File -FilePath $versionPath -NoNewline + } } InitializeCustomSDKToolset diff --git a/eng/restore-toolset.sh b/eng/restore-toolset.sh index f7ba940aa92f..bf7cd57bfaf2 100755 --- a/eng/restore-toolset.sh +++ b/eng/restore-toolset.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# Shared dotnetup acquisition helpers (architecture detection, cache freshness, download). +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/dotnetup-shared.sh" + function InitializeCustomSDKToolset { if [[ "$restore" != true ]]; then return @@ -21,32 +24,206 @@ function InitializeCustomSDKToolset { InitializeDotNetCli true - InstallDotNetSharedFramework "6.0.0" - InstallDotNetSharedFramework "7.0.0" - InstallDotNetSharedFramework "8.0.0" - InstallDotNetSharedFramework "9.0.0" + # Redirect dotnetup data directory under artifacts so build scripts + # don't read/write the user's home-folder manifest. + export DOTNET_DOTNETUP_DATA_DIR="$artifacts_dir/.dotnetup" + + # The following shared frameworks are only needed for testing. + # Set DOTNET_INSTALL_TEST_RUNTIMES=false to skip (e.g. cross-build containers with limited disk). + if [[ "${DOTNET_INSTALL_TEST_RUNTIMES:-true}" != "false" ]]; then + local runtime_specs=("6.0" "7.0" "8.0" "9.0" "10.0") + # Also install the exact runtime versions that arcade's toolset requires + # (from Version.Details.props) so tests can target those specific versions. + local runtime_version + runtime_version=$(ReadVersionDetailsProperty "MicrosoftNETCoreAppRefPackageVersion") + local aspnetcore_version + aspnetcore_version=$(ReadVersionDetailsProperty "MicrosoftAspNetCoreAppRefPackageVersion") + if [[ -n "$runtime_version" ]]; then + runtime_specs+=("$runtime_version") + fi + if [[ -n "$aspnetcore_version" ]]; then + runtime_specs+=("aspnetcore@$aspnetcore_version") + fi + + local native_arch + native_arch=$(GetNativeMachineArchitecture) + if [[ -n "${TARGET_ARCHITECTURE:-}" && "$TARGET_ARCHITECTURE" != "$native_arch" ]]; then + # Cross-build (e.g. an x64 host producing an arm64 test payload). The host cannot execute + # target-architecture runtimes, so installing them into the host .dotnet would break host + # tools that must load a shared framework there (e.g. the NuGet credential provider, whose + # libhostpolicy load fails on an architecture mismatch). Instead, download the + # target-architecture test runtimes into a sidecar folder under artifacts. The matching + # OverlayCrossArchTestRuntimes target in src/Layout/redist/targets/OverlaySdkOnLKG.targets + # copies these into the test host that ships to Helix, where they run on target-architecture + # hardware. The host .dotnet keeps only host-architecture runtimes; host tools roll forward + # to the host SDK runtime. + local sidecar_dir="$artifacts_dir/test-runtimes/$TARGET_ARCHITECTURE" + echo "Cross-build detected (host '$native_arch', target '$TARGET_ARCHITECTURE'). Installing target-architecture test runtimes into sidecar '$sidecar_dir' for the Helix test payload." + mkdir -p "$sidecar_dir" + InstallDotNetSharedFrameworks "$sidecar_dir" "$TARGET_ARCHITECTURE" "${runtime_specs[@]}" + else + InstallDotNetSharedFrameworks "$DOTNET_INSTALL_DIR" "" "${runtime_specs[@]}" + fi + fi CreateBuildEnvScript } -# Installs additional shared frameworks for testing purposes -function InstallDotNetSharedFramework { - local version=$1 - local dotnet_root=$DOTNET_INSTALL_DIR - local fx_dir="$dotnet_root/shared/Microsoft.NETCore.App/$version" +function ReadVersionDetailsProperty { + local property_name=$1 + sed -n "s:.*<$property_name>\([^<]*\).*:\1:p" "$repo_root/eng/Version.Details.props" | head -n 1 +} + +# Maps a dotnetup component (aspnetcore/windowsdesktop/dotnet) to the name of +# its shared-framework folder under /shared. +function GetSharedFrameworkName { + local component=$1 + case "$component" in + aspnetcore) echo "Microsoft.AspNetCore.App" ;; + windowsdesktop) echo "Microsoft.WindowsDesktop.App" ;; + *) echo "Microsoft.NETCore.App" ;; + esac +} + +# Returns the shared-framework directory for a component +# (e.g. /shared/Microsoft.AspNetCore.App). +function GetSharedFrameworkPath { + local dotnet_root=$1 + local component=$2 + echo "$dotnet_root/shared/$(GetSharedFrameworkName "$component")" +} + +# Returns success (0) if a shared framework matching $version (a major.minor +# channel such as 6.0 or an exact version) is already present for $component. +function IsSharedFrameworkInstalled { + local dotnet_root=$1 + local component=$2 + local version=$3 + local fx_root + fx_root="$(GetSharedFrameworkPath "$dotnet_root" "$component")" + + # Only a major.minor channel (e.g. 6.0) should match any patch via a glob. An + # exact version must match an exact folder so that, for example, 8.0.1 does not + # spuriously match an installed 8.0.10. + if [[ "$version" =~ ^[0-9]+\.[0-9]+$ ]]; then + compgen -G "$fx_root/$version*" > /dev/null 2>&1 + else + [[ -d "$fx_root/$version" ]] + fi +} + +# Installs additional shared frameworks for testing purposes. +function InstallDotNetSharedFrameworks { + local dotnet_root=$1 + local arch=$2 + shift 2 + local specs_to_install=() - if [[ ! -d "$fx_dir" ]]; then - GetDotNetInstallScript "$dotnet_root" - local install_script=$_GetDotNetInstallScript + for spec in "$@"; do + # Accept either a dotnet runtime version/channel or a component@version spec + # such as aspnetcore@11.0.0-preview.6. Treat major.minor channels as present + # if any matching patch (e.g. 6.0.36) exists. + local component="dotnet" + local version="$spec" + if [[ "$spec" == *@* ]]; then + component="${spec%@*}" + version="${spec#*@}" + fi + + if ! IsSharedFrameworkInstalled "$dotnet_root" "$component" "$version"; then + specs_to_install+=("$spec") + fi + done - bash "$install_script" --version $version --install-dir "$dotnet_root" --runtime "dotnet" --skip-non-versioned-files - local lastexitcode=$? + if [[ ${#specs_to_install[@]} -eq 0 ]]; then + return + fi + + # dotnetup installs runtimes for its own process architecture and has no + # architecture override (InstallerUtilities.GetDefaultInstallArchitecture uses + # RuntimeInformation.ProcessArchitecture). On a cross-build (e.g. an x64 host + # producing an arm64 test payload), dotnetup would silently install the host + # architecture, so the test runtimes would not match the target Helix queue. + # When a specific architecture is requested, use the dotnet-install script + # directly since it honors --architecture. + if [[ -n "$arch" ]]; then + InstallDotNetSharedFrameworksWithInstallScript "$dotnet_root" "$arch" "${specs_to_install[@]}" + return + fi + + local script_dir + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + local dotnetup_dir="$script_dir/dotnetup" + local dotnetup_exe="$dotnetup_dir/dotnetup" + + if ! ShouldUseCachedDotnetup "$dotnetup_exe"; then + if ! AcquireDotnetup "$dotnetup_dir"; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnetup; falling back to dotnet install script." + InstallDotNetSharedFrameworksWithInstallScript "$dotnet_root" "$arch" "${specs_to_install[@]}" + return + fi + fi + + RunWithoutErrexit "$dotnetup_exe" runtime install "${specs_to_install[@]}" --install-path "$dotnet_root" --set-default-install false --untracked --interactive false + local lastexitcode=$_RunWithoutErrexit + + if [[ $lastexitcode != 0 ]]; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install shared frameworks (${specs_to_install[*]}) to '$dotnet_root' using dotnetup (exit code '$lastexitcode'); falling back to dotnet install script." + InstallDotNetSharedFrameworksWithInstallScript "$dotnet_root" "$arch" "${specs_to_install[@]}" + fi +} + +function InstallDotNetSharedFrameworksWithInstallScript { + local dotnet_root=$1 + local arch=$2 + shift 2 + + GetDotNetInstallScript "$dotnet_root" + local install_script=$_GetDotNetInstallScript + + for spec in "$@"; do + local component="dotnet" + local version="$spec" + if [[ "$spec" == *@* ]]; then + component="${spec%@*}" + version="${spec#*@}" + fi + # Map dotnetup channel (e.g. "9.0") to the specific version the install + # script's --version parameter expects (e.g. "9.0.0"). + local install_version="$version" + if [[ "$install_version" =~ ^[0-9]+\.[0-9]+$ ]]; then + install_version="$install_version.0" + fi + + local install_args=(--version "$install_version" --install-dir "$dotnet_root" --runtime "$component" --skip-non-versioned-files) + if [[ -n "$arch" ]]; then + install_args+=(--architecture "$arch") + fi + + # Disable errexit around the install-script call so the exit-code and filesystem checks below always run. + RunWithoutErrexit bash "$install_script" "${install_args[@]}" + local lastexitcode=$_RunWithoutErrexit + + # Ensure the download was actually successful to some degree. + local framework_installed=false + if IsSharedFrameworkInstalled "$dotnet_root" "$component" "$version"; then + framework_installed=true + fi + + # Promote a false success (exit 0 but nothing on disk) to a real failure. + if [[ $lastexitcode == 0 && "$framework_installed" != true ]]; then + lastexitcode=1 + fi if [[ $lastexitcode != 0 ]]; then - echo "Failed to install Shared Framework $version to '$dotnet_root' (exit code '$lastexitcode')." + local architecture_message="" + if [[ -n "$arch" ]]; then + architecture_message=" for architecture '$arch'" + fi + echo "Failed to install shared framework spec '$spec' to '$dotnet_root' using dotnet install script${architecture_message} (exit code '$lastexitcode', installed '$framework_installed')." ExitWithExitCode $lastexitcode fi - fi + done } function CreateBuildEnvScript { @@ -54,7 +231,6 @@ function CreateBuildEnvScript { scriptPath="$artifacts_dir/sdk-build-env.sh" scriptContents=" #!/usr/bin/env bash -export DOTNET_MULTILEVEL_LOOKUP=0 export DOTNET_ROOT=$DOTNET_INSTALL_DIR export DOTNET_MSBUILD_SDK_RESOLVER_CLI_DIR=$DOTNET_INSTALL_DIR @@ -65,6 +241,7 @@ export DOTNET_ADD_GLOBAL_TOOLS_TO_PATH=0 " echo "$scriptContents" > ${scriptPath} + chmod +x ${scriptPath} } # ReadVersionFromJson [json key] @@ -88,10 +265,10 @@ function CleanOutStage0ToolsetsAndRuntimes { local dotnetSdkVersion=$_ReadGlobalVersion local dotnetRoot=$DOTNET_INSTALL_DIR local versionPath="$dotnetRoot/.version" - local majorVersion="${dotnetSdkVersion:0:1}" - local aspnetRuntimePath="$dotnetRoot/shared/Microsoft.AspNetCore.App/$majorVersion.*" - local coreRuntimePath="$dotnetRoot/shared/Microsoft.NETCore.App/$majorVersion.*" - local wdRuntimePath="$dotnetRoot/shared/Microsoft.WindowsDesktop.App/$majorVersion.*" + local majorVersion="${dotnetSdkVersion%%.*}" + local aspnetRuntimePath="$(GetSharedFrameworkPath "$dotnetRoot" aspnetcore)/$majorVersion.*" + local coreRuntimePath="$(GetSharedFrameworkPath "$dotnetRoot" dotnet)/$majorVersion.*" + local wdRuntimePath="$(GetSharedFrameworkPath "$dotnetRoot" windowsdesktop)/$majorVersion.*" local sdkPath="$dotnetRoot/sdk/*" if [ -f "$versionPath" ]; then diff --git a/eng/sdk-tools.ps1 b/eng/sdk-tools.ps1 new file mode 100644 index 000000000000..afa988bf8c86 --- /dev/null +++ b/eng/sdk-tools.ps1 @@ -0,0 +1,42 @@ +# General-purpose shared helpers for the SDK repo's build/test scripts. +# Dot-source this file to reuse the functions below; it defines functions only +# and has no top-level side effects, so it is safe to dot-source multiple times. +# +# This is the repo-owned counterpart to arcade's eng/common/tools.ps1: put shared +# logic that is NOT specific to a single feature here. (eng/common is managed by +# arcade and changes there are overwritten, so it cannot host repo-owned helpers.) + +# Maps a System.Runtime.InteropServices.Architecture enum value to the lowercase +# dotnet RID architecture token (e.g. "x64", "arm64"). Unknown values map to "x64". +function ConvertTo-RidArchitecture([System.Runtime.InteropServices.Architecture]$Architecture) { + switch ($Architecture) { + ([System.Runtime.InteropServices.Architecture]::Arm64) { return "arm64" } + ([System.Runtime.InteropServices.Architecture]::X86) { return "x86" } + ([System.Runtime.InteropServices.Architecture]::Arm) { return "arm" } + default { return "x64" } + } +} + +# Detect native OS architecture, which may differ from the process architecture +# (e.g., x64 process running on ARM64 Windows via emulation). +function Get-NativeMachineArchitecture { + try { + return ConvertTo-RidArchitecture ([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture) + } + catch { + # Fallback for environments where RuntimeInformation is unavailable + return "x64" + } +} + +# Detect the current process architecture, which may differ from the native OS +# architecture when running under emulation (e.g., an x64 process on ARM64). +function Get-ProcessMachineArchitecture { + try { + return ConvertTo-RidArchitecture ([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture) + } + catch { + # Fallback for environments where RuntimeInformation is unavailable + return "x64" + } +} diff --git a/eng/sdk-tools.sh b/eng/sdk-tools.sh new file mode 100644 index 000000000000..190fa20704b0 --- /dev/null +++ b/eng/sdk-tools.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# General-purpose shared helpers for the SDK repo's build/test scripts. +# Dot-source this file to reuse the functions below; it defines functions only +# and has no top-level side effects, so it is safe to source multiple times. +# +# This is the repo-owned counterpart to arcade's eng/common/tools.sh: put shared +# logic that is NOT specific to a single feature here. (eng/common is managed by +# arcade and changes there are overwritten, so it cannot host repo-owned helpers.) + +# Detect native machine architecture, handling macOS Rosetta 2 +# where uname -m may report x86_64 on arm64 hardware. +function GetNativeMachineArchitecture { + if [[ "$(uname)" == "Darwin" ]] && [[ "$(sysctl -n hw.optional.arm64 2>/dev/null)" == "1" ]]; then + echo "arm64" + return + fi + case "$(uname -m)" in + arm64|aarch64) echo "arm64" ;; + amd64|x86_64) echo "x64" ;; + armv*l) echo "arm" ;; + i[3-6]86) echo "x86" ;; + *) echo "x64" ;; + esac +} diff --git a/src/Layout/redist/targets/OverlaySdkOnLKG.targets b/src/Layout/redist/targets/OverlaySdkOnLKG.targets index 2ff5cc9bf671..e7bf4d7364da 100644 --- a/src/Layout/redist/targets/OverlaySdkOnLKG.targets +++ b/src/Layout/redist/targets/OverlaySdkOnLKG.targets @@ -126,4 +126,29 @@ DestinationFiles="@(WorkloadPackContent->'$(TestHostDotNetRoot)packs\%(RecursiveDir)%(Filename)%(Extension)')" /> + + + + <_CrossArchTestRuntimeSidecar>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'test-runtimes', '$(TargetArchitecture)')) + + + <_CrossArchTestRuntimeFile Include="$(_CrossArchTestRuntimeSidecar)shared\**\*" /> + + + + diff --git a/test/Microsoft.NET.Publish.Tests/GivenThatAPublishedDepsJsonShouldContainVersionInformation.cs b/test/Microsoft.NET.Publish.Tests/GivenThatAPublishedDepsJsonShouldContainVersionInformation.cs index 0df3fe1ace32..7c8359b59c67 100644 --- a/test/Microsoft.NET.Publish.Tests/GivenThatAPublishedDepsJsonShouldContainVersionInformation.cs +++ b/test/Microsoft.NET.Publish.Tests/GivenThatAPublishedDepsJsonShouldContainVersionInformation.cs @@ -141,7 +141,18 @@ public static void Main() var exePath = Path.Combine(publishDirectory.FullName, testProject.Name + ".dll"); - string rollForwardVersion = "8.0.0"; + // Find the actual installed 8.0.x runtime version. With dotnetup, only the latest patch + // (e.g. 8.0.22) may be installed rather than 8.0.0, so we need to discover it dynamically. + string dotnetRoot = SdkTestContext.Current.ToolsetUnderTest.DotNetRoot; + string sharedFxDir = Path.Combine(dotnetRoot, "shared", "Microsoft.NETCore.App"); + string rollForwardVersion = Directory.Exists(sharedFxDir) + ? Directory.GetDirectories(sharedFxDir, "8.0.*") + .Select(Path.GetFileName) + .Where(v => !string.IsNullOrEmpty(v) && Version.TryParse(v, out _)) + .OrderByDescending(v => Version.Parse(v)) + .FirstOrDefault() + : null + ?? "8.0.0"; var runAppCommand = new DotnetCommand(Log, "exec", "--fx-version", rollForwardVersion, exePath);