[main] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -3,9 +3,9 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="10.0.0-beta.24504.4">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="10.0.0-beta.24564.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>f209a925b15bc66ecb9a8825bd9595937bbe3aa1</Sha>
+      <Sha>c1852b9ac37df9a86630c2f078dbee43f7b186e7</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/build.sh

@@ -231,7 +231,6 @@ function Build {
     /p:Restore=$restore \
     /p:Build=$build \
     /p:DotNetBuildRepo=$product_build \
-    /p:ArcadeBuildFromSource=$source_build \
     /p:DotNetBuildSourceOnly=$source_build \
     /p:Rebuild=$rebuild \
     /p:Test=$test \

eng/common/core-templates/job/job.yml

@@ -128,33 +128,11 @@ jobs:
       - ${{ preStep }}
 
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
-      - task: MicroBuildSigningPlugin@4
-        displayName: Install MicroBuild plugin
-        inputs:
-          signType: $(_SignType)
-          zipSources: false
-          feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
-          ${{ if and(eq(parameters.enableMicrobuildForMacAndLinux, 'true'), ne(variables['Agent.Os'], 'Windows_NT')) }}:
-            azureSubscription: 'MicroBuild Signing Task (DevDiv)'
-        env:
-          TeamName: $(_TeamName)
-          MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)'
-          SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+    - template: /eng/common/core-templates/steps/install-microbuild.yml
+      parameters:
+        enableMicrobuild: ${{ parameters.enableMicrobuild }}
+        enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }}
         continueOnError: ${{ parameters.continueOnError }}
-        condition: and(
-          succeeded(),
-          or(
-            and(
-              eq(variables['Agent.Os'], 'Windows_NT'),
-              in(variables['_SignType'], 'real', 'test')
-            ),
-            and(
-              ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }},
-              ne(variables['Agent.Os'], 'Windows_NT'),
-              eq(variables['_SignType'], 'real')
-            )
-          ))
 
   - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}:
     - task: NuGetAuthenticate@1
@@ -183,26 +161,12 @@ jobs:
   - ${{ each step in parameters.componentGovernanceSteps }}:
     - ${{ step }}
 
-  - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
-    - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-      - task: MicroBuildCleanup@1
-        displayName: Execute Microbuild cleanup tasks
-        condition: and(
-          always(),
-          or(
-            and(
-              eq(variables['Agent.Os'], 'Windows_NT'),
-              in(variables['_SignType'], 'real', 'test')
-            ),
-            and(
-              ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }},
-              ne(variables['Agent.Os'], 'Windows_NT'),
-              eq(variables['_SignType'], 'real')
-            )
-          ))
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - template: /eng/common/core-templates/steps/cleanup-microbuild.yml
+      parameters:
+        enableMicrobuild: ${{ parameters.enableMicrobuild }}
+        enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }}
         continueOnError: ${{ parameters.continueOnError }}
-        env:
-          TeamName: $(_TeamName)
 
   # Publish test results
   - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:

eng/common/core-templates/job/source-build.yml

@@ -12,9 +12,10 @@ parameters:
   #   The name of the job. This is included in the job ID.
   # targetRID: ''
   #   The name of the target RID to use, instead of the one auto-detected by Arcade.
-  # nonPortable: false
+  # portableBuild: false
   #   Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
-  #   linux-x64), and compiling against distro-provided packages rather than portable ones.
+  #   linux-x64), and compiling against distro-provided packages rather than portable ones. The
+  #   default is portable mode.
   # skipPublishValidation: false
   #   Disables publishing validation.  By default, a check is performed to ensure no packages are
   #   published by source-build.

eng/common/core-templates/steps/cleanup-microbuild.yml

@@ -0,0 +1,28 @@
+parameters:
+  # Enable cleanup tasks for MicroBuild
+  enableMicrobuild: false
+  # Enable cleanup tasks for MicroBuild on Mac and Linux
+  # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT'
+  enableMicrobuildForMacAndLinux: false
+  continueOnError: false
+
+steps:
+  - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+    - task: MicroBuildCleanup@1
+      displayName: Execute Microbuild cleanup tasks
+      condition: and(
+        always(),
+        or(
+          and(
+            eq(variables['Agent.Os'], 'Windows_NT'),
+            in(variables['_SignType'], 'real', 'test')
+          ),
+          and(
+            ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }},
+            ne(variables['Agent.Os'], 'Windows_NT'),
+            eq(variables['_SignType'], 'real')
+          )
+        ))
+      continueOnError: ${{ parameters.continueOnError }}
+      env:
+        TeamName: $(_TeamName)

eng/common/core-templates/steps/install-microbuild.yml

@@ -0,0 +1,43 @@
+parameters:
+  # Enable cleanup tasks for MicroBuild
+  enableMicrobuild: false
+  # Enable cleanup tasks for MicroBuild on Mac and Linux
+  # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT'
+  enableMicrobuildForMacAndLinux: false
+  continueOnError: false
+
+steps:
+  - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+    # Remove Python downgrade with https://github.com/dotnet/arcade/issues/15151
+    - ${{ if and(eq(parameters.enableMicrobuildForMacAndLinux, 'true'), ne(variables['Agent.Os'], 'Windows_NT')) }}:
+      - task: UsePythonVersion@0
+        displayName: 'Use Python 3.11.x'
+        inputs:
+          versionSpec: '3.11.x'
+
+    - task: MicroBuildSigningPlugin@4
+      displayName: Install MicroBuild plugin
+      inputs:
+        signType: $(_SignType)
+        zipSources: false
+        feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
+        ${{ if and(eq(parameters.enableMicrobuildForMacAndLinux, 'true'), ne(variables['Agent.Os'], 'Windows_NT')) }}:
+          azureSubscription: 'MicroBuild Signing Task (DevDiv)'
+      env:
+        TeamName: $(_TeamName)
+        MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)'
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+      continueOnError: ${{ parameters.continueOnError }}
+      condition: and(
+        succeeded(),
+        or(
+          and(
+            eq(variables['Agent.Os'], 'Windows_NT'),
+            in(variables['_SignType'], 'real', 'test')
+          ),
+          and(
+            ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }},
+            ne(variables['Agent.Os'], 'Windows_NT'),
+            eq(variables['_SignType'], 'real')
+          )
+        ))

eng/common/core-templates/steps/source-build.yml

@@ -76,6 +76,11 @@ steps:
       assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml
     fi
 
+    portableBuildArgs=
+    if [ '${{ parameters.platform.portableBuild }}' != '' ]; then
+      portableBuildArgs='/p:PortabelBuild=${{ parameters.platform.portableBuild }}'
+    fi
+
     ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
       --configuration $buildConfig \
       --restore --build --pack $publishArgs -bl \
@@ -85,8 +90,7 @@ steps:
       $targetRidArgs \
       $runtimeOsArgs \
       $baseOsArgs \
-      /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
-      /p:ArcadeBuildFromSource=true \
+      $portableBuildArgs \
       /p:DotNetBuildSourceOnly=true \
       /p:DotNetBuildRepo=true \
       /p:AssetManifestFileName=$assetManifestFileName

eng/common/native/install-dependencies.sh

@@ -0,0 +1,61 @@
+#!/bin/sh
+
+set -e
+
+# This is a simple script primarily used for CI to install necessary dependencies
+#
+# Usage:
+#
+# ./install-dependencies.sh <OS>
+
+os="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
+
+if [ -z "$os" ]; then
+    . "$(dirname "$0")"/init-os-and-arch.sh
+fi
+
+case "$os" in
+    linux)
+        if [ -e /etc/os-release ]; then
+            . /etc/os-release
+        fi
+
+        if [ "$ID" = "debian" ] || [ "$ID_LIKE" = "debian" ]; then
+            apt update
+
+            apt install -y build-essential gettext locales cmake llvm clang lld lldb liblldb-dev libunwind8-dev libicu-dev liblttng-ust-dev \
+                libssl-dev libkrb5-dev zlib1g-dev pigz cpio
+
+            localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
+        elif [ "$ID" = "fedora" ]; then
+            dnf install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel zlib-devel lttng-ust-devel pigz cpio
+        elif [ "$ID" = "alpine" ]; then
+            apk add build-base cmake bash curl clang llvm-dev lld lldb krb5-dev lttng-ust-dev icu-dev zlib-dev openssl-dev pigz cpio
+        else
+            echo "Unsupported distro. distro: $ID"
+            exit 1
+        fi
+        ;;
+
+    osx|maccatalyst|ios|iossimulator|tvos|tvossimulator)
+        echo "Installed xcode version: $(xcode-select -p)"
+
+        export HOMEBREW_NO_INSTALL_CLEANUP=1
+        export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
+        # Skip brew update for now, see https://github.com/actions/setup-python/issues/577
+        # brew update --preinstall
+        brew bundle --no-upgrade --no-lock --file=- <<EOF
+brew "cmake"
+brew "icu4c"
+brew "openssl@3"
+brew "pkg-config"
+brew "python3"
+brew "pigz"
+EOF
+        ;;
+
+    *)
+        echo "Unsupported platform. OS: $os"
+        exit 1
+        ;;
+esac

eng/common/sdk-task.ps1

@@ -64,7 +64,7 @@ try {
       $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
     }
     if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
-      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.10.0-pre.4.0" -MemberType NoteProperty
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.12.0" -MemberType NoteProperty
     }
     if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
         $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true

eng/common/tools.ps1

@@ -383,8 +383,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
 
   # If the version of msbuild is going to be xcopied,
   # use this version. Version matches a package here:
-  # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.10.0-pre.4.0
-  $defaultXCopyMSBuildVersion = '17.10.0-pre.4.0'
+  # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.12.0
+  $defaultXCopyMSBuildVersion = '17.12.0'
 
   if (!$vsRequirements) {
     if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
@@ -604,14 +604,7 @@ function InitializeBuildTool() {
     }
     $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
 
-    # Use override if it exists - commonly set by source-build
-    if ($null -eq $env:_OverrideArcadeInitializeBuildToolFramework) {
-      $initializeBuildToolFramework="net9.0"
-    } else {
-      $initializeBuildToolFramework=$env:_OverrideArcadeInitializeBuildToolFramework
-    }
-
-    $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = $initializeBuildToolFramework }
+    $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net' }
   } elseif ($msbuildEngine -eq "vs") {
     try {
       $msbuildPath = InitializeVisualStudioMSBuild -install:$restore
@@ -620,7 +613,7 @@ function InitializeBuildTool() {
       ExitWithExitCode 1
     }
 
-    $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS }
+    $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "netframework"; ExcludePrereleaseVS = $excludePrereleaseVS }
   } else {
     Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'."
     ExitWithExitCode 1
@@ -779,8 +772,10 @@ function MSBuild() {
       # new scripts need to work with old packages, so we need to look for the old names/versions
       (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')),
       (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
-      (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.ArcadeLogging.dll')),
-      (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll')),
+
+      # This list doesn't need to be updated anymore and can eventually be removed.
+      (Join-Path $basePath (Join-Path net9.0 'Microsoft.DotNet.ArcadeLogging.dll')),
+      (Join-Path $basePath (Join-Path net9.0 'Microsoft.DotNet.Arcade.Sdk.dll')),
       (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.ArcadeLogging.dll')),
       (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.Arcade.Sdk.dll'))
     )

eng/common/tools.sh

@@ -339,12 +339,6 @@ function InitializeBuildTool {
   # return values
   _InitializeBuildTool="$_InitializeDotNetCli/dotnet"
   _InitializeBuildToolCommand="msbuild"
-  # use override if it exists - commonly set by source-build
-  if [[ "${_OverrideArcadeInitializeBuildToolFramework:-x}" == "x" ]]; then
-    _InitializeBuildToolFramework="net9.0"
-  else
-    _InitializeBuildToolFramework="${_OverrideArcadeInitializeBuildToolFramework}"
-  fi
 }
 
 # Set RestoreNoHttpCache as a workaround for https://github.com/NuGet/Home/issues/3116
@@ -454,10 +448,12 @@ function MSBuild {
     # new scripts need to work with old packages, so we need to look for the old names/versions
     local selectedPath=
     local possiblePaths=()
-    possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" )
-    possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" )
-    possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.ArcadeLogging.dll" )
-    possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.Arcade.Sdk.dll" )
+    possiblePaths+=( "$toolset_dir/net/Microsoft.DotNet.ArcadeLogging.dll" )
+    possiblePaths+=( "$toolset_dir/net/Microsoft.DotNet.Arcade.Sdk.dll" )
+
+    # This list doesn't need to be updated anymore and can eventually be removed.
+    possiblePaths+=( "$toolset_dir/net9.0/Microsoft.DotNet.ArcadeLogging.dll" )
+    possiblePaths+=( "$toolset_dir/net9.0/Microsoft.DotNet.Arcade.Sdk.dll" )
     possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.ArcadeLogging.dll" )
     possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.Arcade.Sdk.dll" )
     for path in "${possiblePaths[@]}"; do

global.json

@@ -1,13 +1,13 @@
 {
   "tools": {
-    "dotnet": "9.0.100-rc.1.24452.12",
+    "dotnet": "9.0.100",
     "runtimes": {
       "dotnet": [
         "8.0.0"
       ]
     }
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.24504.4"
+    "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.24564.1"
   }
 }