diff --git a/.editorconfig b/.editorconfig
index 4db13fd6..d7f84a03 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -13,6 +13,9 @@ indent_style = space
insert_final_newline = true
trim_trailing_whitespace = true
+[*.{sh}]
+end_of_line = lf
+
[*.{csproj,json,props,ruleset,targets}]
indent_size = 2
@@ -152,3 +155,5 @@ csharp_space_between_method_call_empty_parameter_list_parentheses = false
# Wrapping preferences
csharp_preserve_single_line_statements = true
csharp_preserve_single_line_blocks = true
+
+dotnet_diagnostic.IDE0058.severity = none
diff --git a/.gitattributes b/.gitattributes
index 8a92b121..06643587 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,7 +1,9 @@
###############################################################################
# Set default behavior to automatically normalize line endings.
###############################################################################
-* text=auto
+* text=auto
+
+*.sh eol=lf
###############################################################################
# Set default behavior for command prompt diff.
diff --git a/Directory.Build.props b/Directory.Build.props
index e0e10e21..68604c1b 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -3,7 +3,7 @@
- net9.0
+ net10.0
latest
$(NoWarn);CA1515;CS1591
true
@@ -65,6 +65,8 @@
$(IsPackable)
+
+ false
diff --git a/Directory.Packages.props b/Directory.Packages.props
index 22e44f5f..6f244caa 100644
--- a/Directory.Packages.props
+++ b/Directory.Packages.props
@@ -1,17 +1,14 @@
-
+
-
-
-
-
-
+
+
+
+
+
-
-
-
diff --git a/eng/Versions.props b/eng/Versions.props
index 39dcb91a..a1ec564a 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -1,14 +1,14 @@
- 9
+ 10
0
0
$(MajorVersion).$(MinorVersion).$(PatchVersion)
- 9.0.0
- preview
-
- Preview $(PreReleaseVersionIteration)
+ 10.0.0
+ rc
+ 2
+ Release Candidate $(PreReleaseVersionIteration)
false
release
true
diff --git a/eng/common/CIBuild.cmd b/eng/common/CIBuild.cmd
index 56c2f25a..ac1f72bf 100644
--- a/eng/common/CIBuild.cmd
+++ b/eng/common/CIBuild.cmd
@@ -1,2 +1,2 @@
@echo off
-powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
\ No newline at end of file
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1
index 5db4ad71..fc8d6180 100644
--- a/eng/common/SetupNugetSources.ps1
+++ b/eng/common/SetupNugetSources.ps1
@@ -7,11 +7,11 @@
# See example call for this script below.
#
# - task: PowerShell@2
-# displayName: Setup Private Feeds Credentials
+# displayName: Setup internal Feeds Credentials
# condition: eq(variables['Agent.OS'], 'Windows_NT')
# inputs:
-# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
-# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+# filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1
+# arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $Env:Token
# env:
# Token: $(dn-bot-dnceng-artifact-feeds-rw)
#
@@ -34,19 +34,28 @@ Set-StrictMode -Version 2.0
. $PSScriptRoot\tools.ps1
+# Adds or enables the package source with the given name
+function AddOrEnablePackageSource($sources, $disabledPackageSources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) {
+ if ($disabledPackageSources -eq $null -or -not (EnableInternalPackageSource -DisabledPackageSources $disabledPackageSources -Creds $creds -PackageSourceName $SourceName)) {
+ AddPackageSource -Sources $sources -SourceName $SourceName -SourceEndPoint $SourceEndPoint -Creds $creds -Username $userName -pwd $Password
+ }
+}
+
# Add source entry to PackageSources
function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) {
$packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")
if ($packageSource -eq $null)
{
+ Write-Host "Adding package source $SourceName"
+
$packageSource = $doc.CreateElement("add")
$packageSource.SetAttribute("key", $SourceName)
$packageSource.SetAttribute("value", $SourceEndPoint)
$sources.AppendChild($packageSource) | Out-Null
}
else {
- Write-Host "Package source $SourceName already present."
+ Write-Host "Package source $SourceName already present and enabled."
}
AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd
@@ -59,6 +68,8 @@ function AddCredential($creds, $source, $username, $pwd) {
return;
}
+ Write-Host "Inserting credential for feed: " $source
+
# Looks for credential configuration for the given SourceName. Create it if none is found.
$sourceElement = $creds.SelectSingleNode($Source)
if ($sourceElement -eq $null)
@@ -91,24 +102,27 @@ function AddCredential($creds, $source, $username, $pwd) {
$passwordElement.SetAttribute("value", $pwd)
}
-function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) {
- $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
-
- Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
-
- ForEach ($PackageSource in $maestroPrivateSources) {
- Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
- AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd
+# Enable all darc-int package sources.
+function EnableMaestroInternalPackageSources($DisabledPackageSources, $Creds) {
+ $maestroInternalSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
+ ForEach ($DisabledPackageSource in $maestroInternalSources) {
+ EnableInternalPackageSource -DisabledPackageSources $DisabledPackageSources -Creds $Creds -PackageSourceName $DisabledPackageSource.key
}
}
-function EnablePrivatePackageSources($DisabledPackageSources) {
- $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
- ForEach ($DisabledPackageSource in $maestroPrivateSources) {
- Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
+# Enables an internal package source by name, if found. Returns true if the package source was found and enabled, false otherwise.
+function EnableInternalPackageSource($DisabledPackageSources, $Creds, $PackageSourceName) {
+ $DisabledPackageSource = $DisabledPackageSources.SelectSingleNode("add[@key='$PackageSourceName']")
+ if ($DisabledPackageSource) {
+ Write-Host "Enabling internal source '$($DisabledPackageSource.key)'."
+
# Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries
$DisabledPackageSources.RemoveChild($DisabledPackageSource)
+
+ AddCredential -Creds $creds -Source $DisabledPackageSource.Key -Username $userName -pwd $Password
+ return $true
}
+ return $false
}
if (!(Test-Path $ConfigFile -PathType Leaf)) {
@@ -121,15 +135,17 @@ $doc = New-Object System.Xml.XmlDocument
$filename = (Get-Item $ConfigFile).FullName
$doc.Load($filename)
-# Get reference to or create one if none exist already
+# Get reference to - fail if none exist
$sources = $doc.DocumentElement.SelectSingleNode("packageSources")
if ($sources -eq $null) {
- $sources = $doc.CreateElement("packageSources")
- $doc.DocumentElement.AppendChild($sources) | Out-Null
+ Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. NuGet config file must contain a packageSources section: $ConfigFile"
+ ExitWithExitCode 1
}
$creds = $null
+$feedSuffix = "v3/index.json"
if ($Password) {
+ $feedSuffix = "v2"
# Looks for a node. Create it if none is found.
$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
if ($creds -eq $null) {
@@ -138,33 +154,22 @@ if ($Password) {
}
}
+$userName = "dn-bot"
+
# Check for disabledPackageSources; we'll enable any darc-int ones we find there
$disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources")
if ($disabledSources -ne $null) {
Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node"
- EnablePrivatePackageSources -DisabledPackageSources $disabledSources
-}
-
-$userName = "dn-bot"
-
-# Insert credential nodes for Maestro's private feeds
-InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password
-
-# 3.1 uses a different feed url format so it's handled differently here
-$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
-if ($dotnet31Source -ne $null) {
- AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
- AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
+ EnableMaestroInternalPackageSources -DisabledPackageSources $disabledSources -Creds $creds
}
-
-$dotnetVersions = @('5','6','7','8','9')
+$dotnetVersions = @('5','6','7','8','9','10')
foreach ($dotnetVersion in $dotnetVersions) {
$feedPrefix = "dotnet" + $dotnetVersion;
$dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']")
if ($dotnetSource -ne $null) {
- AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
- AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
+ AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password
+ AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password
}
}
diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh
index 4604b61b..b97cc536 100755
--- a/eng/common/SetupNugetSources.sh
+++ b/eng/common/SetupNugetSources.sh
@@ -11,8 +11,8 @@
# - task: Bash@3
# displayName: Setup Internal Feeds
# inputs:
-# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
-# arguments: $(Build.SourcesDirectory)/NuGet.config
+# filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh
+# arguments: $(System.DefaultWorkingDirectory)/NuGet.config
# condition: ne(variables['Agent.OS'], 'Windows_NT')
# - task: NuGetAuthenticate@1
#
@@ -52,78 +52,124 @@ if [[ `uname -s` == "Darwin" ]]; then
TB=''
fi
-# Ensure there is a ... section.
-grep -i "" $ConfigFile
-if [ "$?" != "0" ]; then
- echo "Adding ... section."
- ConfigNodeHeader=""
- PackageSourcesTemplate="${TB}${NL}${TB}"
+# Enables an internal package source by name, if found. Returns 0 if found and enabled, 1 if not found.
+EnableInternalPackageSource() {
+ local PackageSourceName="$1"
+
+ # Check if disabledPackageSources section exists
+ grep -i "" "$ConfigFile" > /dev/null
+ if [ "$?" != "0" ]; then
+ return 1 # No disabled sources section
+ fi
+
+ # Check if this source name is disabled
+ grep -i " /dev/null
+ if [ "$?" == "0" ]; then
+ echo "Enabling internal source '$PackageSourceName'."
+ # Remove the disabled entry (including any surrounding comments or whitespace on the same line)
+ sed -i.bak "//d" "$ConfigFile"
+
+ # Add the source name to PackageSources for credential handling
+ PackageSources+=("$PackageSourceName")
+ return 0 # Found and enabled
+ fi
+
+ return 1 # Not found in disabled sources
+}
+
+# Add source entry to PackageSources
+AddPackageSource() {
+ local SourceName="$1"
+ local SourceEndPoint="$2"
+
+ # Check if source already exists
+ grep -i " /dev/null
+ if [ "$?" == "0" ]; then
+ echo "Package source $SourceName already present and enabled."
+ PackageSources+=("$SourceName")
+ return
+ fi
+
+ echo "Adding package source $SourceName"
+ PackageSourcesNodeFooter=""
+ PackageSourceTemplate="${TB}"
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" "$ConfigFile"
+ PackageSources+=("$SourceName")
+}
+
+# Adds or enables the package source with the given name
+AddOrEnablePackageSource() {
+ local SourceName="$1"
+ local SourceEndPoint="$2"
+
+ # Try to enable if disabled, if not found then add new source
+ EnableInternalPackageSource "$SourceName"
+ if [ "$?" != "0" ]; then
+ AddPackageSource "$SourceName" "$SourceEndPoint"
+ fi
+}
- sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" $ConfigFile
-fi
+# Enable all darc-int package sources
+EnableMaestroInternalPackageSources() {
+ # Check if disabledPackageSources section exists
+ grep -i "" "$ConfigFile" > /dev/null
+ if [ "$?" != "0" ]; then
+ return # No disabled sources section
+ fi
+
+ # Find all darc-int disabled sources
+ local DisabledDarcIntSources=()
+ DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' "$ConfigFile" | tr -d '"')
+
+ for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do
+ if [[ $DisabledSourceName == darc-int* ]]; then
+ EnableInternalPackageSource "$DisabledSourceName"
+ fi
+ done
+}
-# Ensure there is a ... section.
-grep -i "" $ConfigFile
+# Ensure there is a ... section.
+grep -i "" $ConfigFile
if [ "$?" != "0" ]; then
- echo "Adding ... section."
-
- PackageSourcesNodeFooter=""
- PackageSourceCredentialsTemplate="${TB}${NL}${TB}"
-
- sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile
+ Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. NuGet config file must contain a packageSources section: $ConfigFile"
+ ExitWithExitCode 1
fi
PackageSources=()
-# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present
-grep -i "... section.
+ grep -i "" $ConfigFile
if [ "$?" != "0" ]; then
- echo "Adding dotnet3.1-internal to the packageSources."
- PackageSourcesNodeFooter=""
- PackageSourceTemplate="${TB}"
+ echo "Adding ... section."
- sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
- fi
- PackageSources+=('dotnet3.1-internal')
-
- grep -i "" $ConfigFile
- if [ "$?" != "0" ]; then
- echo "Adding dotnet3.1-internal-transport to the packageSources."
PackageSourcesNodeFooter=""
- PackageSourceTemplate="${TB}"
+ PackageSourceCredentialsTemplate="${TB}${NL}${TB}"
- sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile
fi
- PackageSources+=('dotnet3.1-internal-transport')
fi
-DotNetVersions=('5' '6' '7' '8' '9')
+# Check for disabledPackageSources; we'll enable any darc-int ones we find there
+grep -i "" $ConfigFile > /dev/null
+if [ "$?" == "0" ]; then
+ echo "Checking for any darc-int disabled package sources in the disabledPackageSources node"
+ EnableMaestroInternalPackageSources
+fi
+
+DotNetVersions=('5' '6' '7' '8' '9' '10')
for DotNetVersion in ${DotNetVersions[@]} ; do
FeedPrefix="dotnet${DotNetVersion}";
- grep -i " /dev/null
if [ "$?" == "0" ]; then
- grep -i ""
-
- sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
- fi
- PackageSources+=("$FeedPrefix-internal")
-
- grep -i "" $ConfigFile
- if [ "$?" != "0" ]; then
- echo "Adding $FeedPrefix-internal-transport to the packageSources."
- PackageSourcesNodeFooter=""
- PackageSourceTemplate="${TB}"
-
- sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
- fi
- PackageSources+=("$FeedPrefix-internal-transport")
+ AddOrEnablePackageSource "$FeedPrefix-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal/nuget/$FeedSuffix"
+ AddOrEnablePackageSource "$FeedPrefix-internal-transport" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal-transport/nuget/$FeedSuffix"
fi
done
@@ -139,29 +185,12 @@ if [ "$CredToken" ]; then
# Check if there is no existing credential for this FeedName
grep -i "<$FeedName>" $ConfigFile
if [ "$?" != "0" ]; then
- echo "Adding credentials for $FeedName."
+ echo " Inserting credential for feed: $FeedName"
PackageSourceCredentialsNodeFooter=""
- NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}"
+ NewCredential="${TB}${TB}<$FeedName>${NL}${TB}${NL}${TB}${TB}${NL}${TB}${TB}"
sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
fi
done
fi
-
-# Re-enable any entries in disabledPackageSources where the feed name contains darc-int
-grep -i "" $ConfigFile
-if [ "$?" == "0" ]; then
- DisabledDarcIntSources=()
- echo "Re-enabling any disabled \"darc-int\" package sources in $ConfigFile"
- DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' $ConfigFile | tr -d '"')
- for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do
- if [[ $DisabledSourceName == darc-int* ]]
- then
- OldDisableValue=""
- NewDisableValue=""
- sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile
- echo "Neutralized disablePackageSources entry for '$DisabledSourceName'"
- fi
- done
-fi
diff --git a/eng/common/build.ps1 b/eng/common/build.ps1
index 438f9920..8cfee107 100644
--- a/eng/common/build.ps1
+++ b/eng/common/build.ps1
@@ -7,6 +7,7 @@ Param(
[string] $msbuildEngine = $null,
[bool] $warnAsError = $true,
[bool] $nodeReuse = $true,
+ [switch] $buildCheck = $false,
[switch][Alias('r')]$restore,
[switch] $deployDeps,
[switch][Alias('b')]$build,
@@ -20,6 +21,7 @@ Param(
[switch] $publish,
[switch] $clean,
[switch][Alias('pb')]$productBuild,
+ [switch]$fromVMR,
[switch][Alias('bl')]$binaryLog,
[switch][Alias('nobl')]$excludeCIBinarylog,
[switch] $ci,
@@ -71,6 +73,9 @@ function Print-Usage() {
Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio"
Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)"
+ Write-Host " -nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')"
+ Write-Host " -buildCheck Sets /check msbuild parameter"
+ Write-Host " -fromVMR Set when building from within the VMR"
Write-Host ""
Write-Host "Command line arguments not listed above are passed thru to msbuild."
@@ -97,6 +102,7 @@ function Build {
$bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' }
$platformArg = if ($platform) { "/p:Platform=$platform" } else { '' }
+ $check = if ($buildCheck) { '/check' } else { '' }
if ($projects) {
# Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons.
@@ -113,6 +119,7 @@ function Build {
MSBuild $toolsetBuildProj `
$bl `
$platformArg `
+ $check `
/p:Configuration=$configuration `
/p:RepoRoot=$RepoRoot `
/p:Restore=$restore `
@@ -122,11 +129,13 @@ function Build {
/p:Deploy=$deploy `
/p:Test=$test `
/p:Pack=$pack `
- /p:DotNetBuildRepo=$productBuild `
+ /p:DotNetBuild=$productBuild `
+ /p:DotNetBuildFromVMR=$fromVMR `
/p:IntegrationTest=$integrationTest `
/p:PerformanceTest=$performanceTest `
/p:Sign=$sign `
/p:Publish=$publish `
+ /p:RestoreStaticGraphEnableBinaryLogger=$binaryLog `
@properties
}
diff --git a/eng/common/build.sh b/eng/common/build.sh
index ac1ee862..9767bb41 100755
--- a/eng/common/build.sh
+++ b/eng/common/build.sh
@@ -42,6 +42,8 @@ usage()
echo " --prepareMachine Prepare machine for CI run, clean up processes after build"
echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')"
echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')"
+ echo " --buildCheck Sets /check msbuild parameter"
+ echo " --fromVMR Set when building from within the VMR"
echo ""
echo "Command line arguments not listed above are passed thru to msbuild."
echo "Arguments can also be passed in with a single hyphen."
@@ -63,6 +65,7 @@ restore=false
build=false
source_build=false
product_build=false
+from_vmr=false
rebuild=false
test=false
integration_test=false
@@ -76,6 +79,7 @@ clean=false
warn_as_error=true
node_reuse=true
+build_check=false
binary_log=false
exclude_ci_binary_log=false
pipelines_log=false
@@ -87,7 +91,7 @@ verbosity='minimal'
runtime_source_feed=''
runtime_source_feed_key=''
-properties=''
+properties=()
while [[ $# > 0 ]]; do
opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
@@ -127,19 +131,22 @@ while [[ $# > 0 ]]; do
-pack)
pack=true
;;
- -sourcebuild|-sb)
+ -sourcebuild|-source-build|-sb)
build=true
source_build=true
product_build=true
restore=true
pack=true
;;
- -productBuild|-pb)
+ -productbuild|-product-build|-pb)
build=true
product_build=true
restore=true
pack=true
;;
+ -fromvmr|-from-vmr)
+ from_vmr=true
+ ;;
-test|-t)
test=true
;;
@@ -173,6 +180,9 @@ while [[ $# > 0 ]]; do
node_reuse=$2
shift
;;
+ -buildcheck)
+ build_check=true
+ ;;
-runtimesourcefeed)
runtime_source_feed=$2
shift
@@ -182,7 +192,7 @@ while [[ $# > 0 ]]; do
shift
;;
*)
- properties="$properties $1"
+ properties+=("$1")
;;
esac
@@ -216,7 +226,7 @@ function Build {
InitializeCustomToolset
if [[ ! -z "$projects" ]]; then
- properties="$properties /p:Projects=$projects"
+ properties+=("/p:Projects=$projects")
fi
local bl=""
@@ -224,15 +234,21 @@ function Build {
bl="/bl:\"$log_dir/Build.binlog\""
fi
+ local check=""
+ if [[ "$build_check" == true ]]; then
+ check="/check"
+ fi
+
MSBuild $_InitializeToolset \
$bl \
+ $check \
/p:Configuration=$configuration \
/p:RepoRoot="$repo_root" \
/p:Restore=$restore \
/p:Build=$build \
- /p:DotNetBuildRepo=$product_build \
- /p:ArcadeBuildFromSource=$source_build \
+ /p:DotNetBuild=$product_build \
/p:DotNetBuildSourceOnly=$source_build \
+ /p:DotNetBuildFromVMR=$from_vmr \
/p:Rebuild=$rebuild \
/p:Test=$test \
/p:Pack=$pack \
@@ -240,7 +256,8 @@ function Build {
/p:PerformanceTest=$performance_test \
/p:Sign=$sign \
/p:Publish=$publish \
- $properties
+ /p:RestoreStaticGraphEnableBinaryLogger=$binary_log \
+ ${properties[@]+"${properties[@]}"}
ExitWithExitCode 0
}
diff --git a/eng/common/cibuild.sh b/eng/common/cibuild.sh
index 1a02c0de..9d9001df 100755
--- a/eng/common/cibuild.sh
+++ b/eng/common/cibuild.sh
@@ -7,10 +7,10 @@ while [[ -h $source ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"
- # if $source was a relative symlink, we need to resolve it relative to the path where
+ # if $source was a relative symlink, we need to resolve it relative to the path where
# the symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
-. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@
\ No newline at end of file
+. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@
diff --git a/eng/common/cross/arm64/tizen/tizen.patch b/eng/common/cross/arm64/tizen/tizen.patch
index af7c8be0..2cebc547 100644
--- a/eng/common/cross/arm64/tizen/tizen.patch
+++ b/eng/common/cross/arm64/tizen/tizen.patch
@@ -5,5 +5,5 @@ diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so
Use the shared library, but some functions are only in
the static library, so try that secondarily. */
OUTPUT_FORMAT(elf64-littleaarch64)
--GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-aarch64.so.1 ) )
+-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib64/ld-linux-aarch64.so.1 ) )
+GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-aarch64.so.1 ) )
diff --git a/eng/common/cross/armel/armel.jessie.patch b/eng/common/cross/armel/armel.jessie.patch
deleted file mode 100644
index 2d261561..00000000
--- a/eng/common/cross/armel/armel.jessie.patch
+++ /dev/null
@@ -1,43 +0,0 @@
-diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h
---- a/usr/include/urcu/uatomic/generic.h 2014-10-22 15:00:58.000000000 -0700
-+++ b/usr/include/urcu/uatomic/generic.h 2020-10-30 21:38:28.550000000 -0700
-@@ -69,10 +69,10 @@
- #endif
- #ifdef UATOMIC_HAS_ATOMIC_SHORT
- case 2:
-- return __sync_val_compare_and_swap_2(addr, old, _new);
-+ return __sync_val_compare_and_swap_2((uint16_t*) addr, old, _new);
- #endif
- case 4:
-- return __sync_val_compare_and_swap_4(addr, old, _new);
-+ return __sync_val_compare_and_swap_4((uint32_t*) addr, old, _new);
- #if (CAA_BITS_PER_LONG == 64)
- case 8:
- return __sync_val_compare_and_swap_8(addr, old, _new);
-@@ -109,7 +109,7 @@
- return;
- #endif
- case 4:
-- __sync_and_and_fetch_4(addr, val);
-+ __sync_and_and_fetch_4((uint32_t*) addr, val);
- return;
- #if (CAA_BITS_PER_LONG == 64)
- case 8:
-@@ -148,7 +148,7 @@
- return;
- #endif
- case 4:
-- __sync_or_and_fetch_4(addr, val);
-+ __sync_or_and_fetch_4((uint32_t*) addr, val);
- return;
- #if (CAA_BITS_PER_LONG == 64)
- case 8:
-@@ -187,7 +187,7 @@
- return __sync_add_and_fetch_2(addr, val);
- #endif
- case 4:
-- return __sync_add_and_fetch_4(addr, val);
-+ return __sync_add_and_fetch_4((uint32_t*) addr, val);
- #if (CAA_BITS_PER_LONG == 64)
- case 8:
- return __sync_add_and_fetch_8(addr, val);
diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh
index 7e9ba2b7..fbd8d808 100755
--- a/eng/common/cross/build-android-rootfs.sh
+++ b/eng/common/cross/build-android-rootfs.sh
@@ -6,10 +6,11 @@ usage()
{
echo "Creates a toolchain and sysroot used for cross-compiling for Android."
echo
- echo "Usage: $0 [BuildArch] [ApiLevel]"
+ echo "Usage: $0 [BuildArch] [ApiLevel] [--ndk NDKVersion]"
echo
echo "BuildArch is the target architecture of Android. Currently only arm64 is supported."
echo "ApiLevel is the target Android API level. API levels usually match to Android releases. See https://source.android.com/source/build-numbers.html"
+ echo "NDKVersion is the version of Android NDK. The default is r21. See https://developer.android.com/ndk/downloads/revision_history"
echo
echo "By default, the toolchain and sysroot will be generated in cross/android-rootfs/toolchain/[BuildArch]. You can change this behavior"
echo "by setting the TOOLCHAIN_DIR environment variable"
@@ -25,10 +26,15 @@ __BuildArch=arm64
__AndroidArch=aarch64
__AndroidToolchain=aarch64-linux-android
-for i in "$@"
- do
- lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")"
- case $lowerI in
+while :; do
+ if [[ "$#" -le 0 ]]; then
+ break
+ fi
+
+ i=$1
+
+ lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")"
+ case $lowerI in
-?|-h|--help)
usage
exit 1
@@ -43,6 +49,10 @@ for i in "$@"
__AndroidArch=arm
__AndroidToolchain=arm-linux-androideabi
;;
+ --ndk)
+ shift
+ __NDK_Version=$1
+ ;;
*[0-9])
__ApiLevel=$i
;;
@@ -50,8 +60,17 @@ for i in "$@"
__UnprocessedBuildArgs="$__UnprocessedBuildArgs $i"
;;
esac
+ shift
done
+if [[ "$__NDK_Version" == "r21" ]] || [[ "$__NDK_Version" == "r22" ]]; then
+ __NDK_File_Arch_Spec=-x86_64
+ __SysRoot=sysroot
+else
+ __NDK_File_Arch_Spec=
+ __SysRoot=toolchains/llvm/prebuilt/linux-x86_64/sysroot
+fi
+
# Obtain the location of the bash script to figure out where the root of the repo is.
__ScriptBaseDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@@ -78,6 +97,7 @@ fi
echo "Target API level: $__ApiLevel"
echo "Target architecture: $__BuildArch"
+echo "NDK version: $__NDK_Version"
echo "NDK location: $__NDK_Dir"
echo "Target Toolchain location: $__ToolchainDir"
@@ -85,8 +105,8 @@ echo "Target Toolchain location: $__ToolchainDir"
if [ ! -d $__NDK_Dir ]; then
echo Downloading the NDK into $__NDK_Dir
mkdir -p $__NDK_Dir
- wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip
- unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__CrossDir
+ wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux$__NDK_File_Arch_Spec.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux.zip
+ unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux.zip -d $__CrossDir
fi
if [ ! -d $__lldb_Dir ]; then
@@ -116,16 +136,11 @@ for path in $(wget -qO- https://packages.termux.dev/termux-main-21/dists/stable/
fi
done
-cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/sysroot/usr/"
+cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/$__SysRoot/usr/"
# Generate platform file for build.sh script to assign to __DistroRid
echo "Generating platform file..."
-echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/android_platform
-
-echo "Now to build coreclr, libraries and installers; run:"
-echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
- --subsetCategory coreclr
-echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
- --subsetCategory libraries
-echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
- --subsetCategory installer
+echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/$__SysRoot/android_platform
+
+echo "Now to build coreclr, libraries and host; run:"
+echo ROOTFS_DIR=$(realpath $__ToolchainDir/$__SysRoot) ./build.sh clr+libs+host --cross --arch $__BuildArch
diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh
index 4b5e8d71..8abfb71f 100755
--- a/eng/common/cross/build-rootfs.sh
+++ b/eng/common/cross/build-rootfs.sh
@@ -5,7 +5,7 @@ set -e
usage()
{
echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir ]"
- echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86"
+ echo "BuildArch can be: arm(default), arm64, armel, armv6, loongarch64, ppc64le, riscv64, s390x, x64, x86"
echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine"
echo " for alpine can be specified with version: alpineX.YY or alpineedge"
echo " for FreeBSD can be: freebsd13, freebsd14"
@@ -15,6 +15,7 @@ usage()
echo "llvmx[.y] - optional, LLVM version for LLVM related packages."
echo "--skipunmount - optional, will skip the unmount of rootfs folder."
echo "--skipsigcheck - optional, will skip package signature checks (allowing untrusted packages)."
+ echo "--skipemulation - optional, will skip qemu and debootstrap requirement when building environment for debian based systems."
echo "--use-mirror - optional, use mirror URL to fetch resources, when available."
echo "--jobs N - optional, restrict to N jobs."
exit 1
@@ -52,28 +53,27 @@ __UbuntuPackages+=" symlinks"
__UbuntuPackages+=" libicu-dev"
__UbuntuPackages+=" liblttng-ust-dev"
__UbuntuPackages+=" libunwind8-dev"
-__UbuntuPackages+=" libnuma-dev"
__AlpinePackages+=" gettext-dev"
__AlpinePackages+=" icu-dev"
__AlpinePackages+=" libunwind-dev"
__AlpinePackages+=" lttng-ust-dev"
__AlpinePackages+=" compiler-rt"
-__AlpinePackages+=" numactl-dev"
# runtime libraries' dependencies
__UbuntuPackages+=" libcurl4-openssl-dev"
__UbuntuPackages+=" libkrb5-dev"
__UbuntuPackages+=" libssl-dev"
__UbuntuPackages+=" zlib1g-dev"
+__UbuntuPackages+=" libbrotli-dev"
__AlpinePackages+=" curl-dev"
__AlpinePackages+=" krb5-dev"
__AlpinePackages+=" openssl-dev"
__AlpinePackages+=" zlib-dev"
-__FreeBSDBase="13.3-RELEASE"
-__FreeBSDPkg="1.17.0"
+__FreeBSDBase="13.4-RELEASE"
+__FreeBSDPkg="1.21.3"
__FreeBSDABI="13"
__FreeBSDPackages="libunwind"
__FreeBSDPackages+=" icu"
@@ -91,18 +91,18 @@ __HaikuPackages="gcc_syslibs"
__HaikuPackages+=" gcc_syslibs_devel"
__HaikuPackages+=" gmp"
__HaikuPackages+=" gmp_devel"
-__HaikuPackages+=" icu66"
-__HaikuPackages+=" icu66_devel"
+__HaikuPackages+=" icu[0-9]+"
+__HaikuPackages+=" icu[0-9]*_devel"
__HaikuPackages+=" krb5"
__HaikuPackages+=" krb5_devel"
__HaikuPackages+=" libiconv"
__HaikuPackages+=" libiconv_devel"
-__HaikuPackages+=" llvm12_libunwind"
-__HaikuPackages+=" llvm12_libunwind_devel"
+__HaikuPackages+=" llvm[0-9]*_libunwind"
+__HaikuPackages+=" llvm[0-9]*_libunwind_devel"
__HaikuPackages+=" mpfr"
__HaikuPackages+=" mpfr_devel"
-__HaikuPackages+=" openssl"
-__HaikuPackages+=" openssl_devel"
+__HaikuPackages+=" openssl3"
+__HaikuPackages+=" openssl3_devel"
__HaikuPackages+=" zlib"
__HaikuPackages+=" zlib_devel"
@@ -128,10 +128,12 @@ __AlpineKeys='
616adfeb:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAq0BFD1D4lIxQcsqEpQzU\npNCYM3aP1V/fxxVdT4DWvSI53JHTwHQamKdMWtEXetWVbP5zSROniYKFXd/xrD9X\n0jiGHey3lEtylXRIPxe5s+wXoCmNLcJVnvTcDtwx/ne2NLHxp76lyc25At+6RgE6\nADjLVuoD7M4IFDkAsd8UQ8zM0Dww9SylIk/wgV3ZkifecvgUQRagrNUdUjR56EBZ\nraQrev4hhzOgwelT0kXCu3snbUuNY/lU53CoTzfBJ5UfEJ5pMw1ij6X0r5S9IVsy\nKLWH1hiO0NzU2c8ViUYCly4Fe9xMTFc6u2dy/dxf6FwERfGzETQxqZvSfrRX+GLj\n/QZAXiPg5178hT/m0Y3z5IGenIC/80Z9NCi+byF1WuJlzKjDcF/TU72zk0+PNM/H\nKuppf3JT4DyjiVzNC5YoWJT2QRMS9KLP5iKCSThwVceEEg5HfhQBRT9M6KIcFLSs\nmFjx9kNEEmc1E8hl5IR3+3Ry8G5/bTIIruz14jgeY9u5jhL8Vyyvo41jgt9sLHR1\n/J1TxKfkgksYev7PoX6/ZzJ1ksWKZY5NFoDXTNYUgzFUTOoEaOg3BAQKadb3Qbbq\nXIrxmPBdgrn9QI7NCgfnAY3Tb4EEjs3ON/BNyEhUENcXOH6I1NbcuBQ7g9P73kE4\nVORdoc8MdJ5eoKBpO8Ww8HECAwEAAQ==
616ae350:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyduVzi1mWm+lYo2Tqt/0\nXkCIWrDNP1QBMVPrE0/ZlU2bCGSoo2Z9FHQKz/mTyMRlhNqTfhJ5qU3U9XlyGOPJ\npiM+b91g26pnpXJ2Q2kOypSgOMOPA4cQ42PkHBEqhuzssfj9t7x47ppS94bboh46\nxLSDRff/NAbtwTpvhStV3URYkxFG++cKGGa5MPXBrxIp+iZf9GnuxVdST5PGiVGP\nODL/b69sPJQNbJHVquqUTOh5Ry8uuD2WZuXfKf7/C0jC/ie9m2+0CttNu9tMciGM\nEyKG1/Xhk5iIWO43m4SrrT2WkFlcZ1z2JSf9Pjm4C2+HovYpihwwdM/OdP8Xmsnr\nDzVB4YvQiW+IHBjStHVuyiZWc+JsgEPJzisNY0Wyc/kNyNtqVKpX6dRhMLanLmy+\nf53cCSI05KPQAcGj6tdL+D60uKDkt+FsDa0BTAobZ31OsFVid0vCXtsbplNhW1IF\nHwsGXBTVcfXg44RLyL8Lk/2dQxDHNHzAUslJXzPxaHBLmt++2COa2EI1iWlvtznk\nOk9WP8SOAIj+xdqoiHcC4j72BOVVgiITIJNHrbppZCq6qPR+fgXmXa+sDcGh30m6\n9Wpbr28kLMSHiENCWTdsFij+NQTd5S47H7XTROHnalYDuF1RpS+DpQidT5tUimaT\nJZDr++FjKrnnijbyNF8b98UCAwEAAQ==
616db30d:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnpUpyWDWjlUk3smlWeA0\nlIMW+oJ38t92CRLHH3IqRhyECBRW0d0aRGtq7TY8PmxjjvBZrxTNDpJT6KUk4LRm\na6A6IuAI7QnNK8SJqM0DLzlpygd7GJf8ZL9SoHSH+gFsYF67Cpooz/YDqWrlN7Vw\ntO00s0B+eXy+PCXYU7VSfuWFGK8TGEv6HfGMALLjhqMManyvfp8hz3ubN1rK3c8C\nUS/ilRh1qckdbtPvoDPhSbTDmfU1g/EfRSIEXBrIMLg9ka/XB9PvWRrekrppnQzP\nhP9YE3x/wbFc5QqQWiRCYyQl/rgIMOXvIxhkfe8H5n1Et4VAorkpEAXdsfN8KSVv\nLSMazVlLp9GYq5SUpqYX3KnxdWBgN7BJoZ4sltsTpHQ/34SXWfu3UmyUveWj7wp0\nx9hwsPirVI00EEea9AbP7NM2rAyu6ukcm4m6ATd2DZJIViq2es6m60AE6SMCmrQF\nwmk4H/kdQgeAELVfGOm2VyJ3z69fQuywz7xu27S6zTKi05Qlnohxol4wVb6OB7qG\nLPRtK9ObgzRo/OPumyXqlzAi/Yvyd1ZQk8labZps3e16bQp8+pVPiumWioMFJDWV\nGZjCmyMSU8V6MB6njbgLHoyg2LCukCAeSjbPGGGYhnKLm1AKSoJh3IpZuqcKCk5C\n8CM1S15HxV78s9dFntEqIokCAwEAAQ==
+66ba20fe:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtfB12w4ZgqsXWZDfUAV/\n6Y4aHUKIu3q4SXrNZ7CXF9nXoAVYrS7NAxJdAodsY3vPCN0g5O8DFXR+390LdOuQ\n+HsGKCc1k5tX5ZXld37EZNTNSbR0k+NKhd9h6X3u6wqPOx7SIKxwAQR8qeeFq4pP\nrt9GAGlxtuYgzIIcKJPwE0dZlcBCg+GnptCUZXp/38BP1eYC+xTXSL6Muq1etYfg\nodXdb7Yl+2h1IHuOwo5rjgY5kpY7GcAs8AjGk3lDD/av60OTYccknH0NCVSmPoXK\nvrxDBOn0LQRNBLcAfnTKgHrzy0Q5h4TNkkyTgxkoQw5ObDk9nnabTxql732yy9BY\ns+hM9+dSFO1HKeVXreYSA2n1ndF18YAvAumzgyqzB7I4pMHXq1kC/8bONMJxwSkS\nYm6CoXKyavp7RqGMyeVpRC7tV+blkrrUml0BwNkxE+XnwDRB3xDV6hqgWe0XrifD\nYTfvd9ScZQP83ip0r4IKlq4GMv/R5shcCRJSkSZ6QSGshH40JYSoiwJf5FHbj9ND\n7do0UAqebWo4yNx63j/wb2ULorW3AClv0BCFSdPsIrCStiGdpgJDBR2P2NZOCob3\nG9uMj+wJD6JJg2nWqNJxkANXX37Qf8plgzssrhrgOvB0fjjS7GYhfkfmZTJ0wPOw\nA8+KzFseBh4UFGgue78KwgkCAwEAAQ==
'
__Keyring=
__KeyringFile="/usr/share/keyrings/ubuntu-archive-keyring.gpg"
__SkipSigCheck=0
+__SkipEmulation=0
__UseMirror=0
__UnprocessedBuildArgs=
@@ -162,9 +164,13 @@ while :; do
armel)
__BuildArch=armel
__UbuntuArch=armel
- __UbuntuRepo="http://ftp.debian.org/debian/"
- __CodeName=jessie
+ __UbuntuRepo="http://archive.debian.org/debian/"
+ __CodeName=buster
__KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
+ __LLDB_Package="liblldb-6.0-dev"
+ __UbuntuPackages="${__UbuntuPackages// libomp-dev/}"
+ __UbuntuPackages="${__UbuntuPackages// libomp5/}"
+ __UbuntuSuites=
;;
armv6)
__BuildArch=armv6
@@ -180,6 +186,18 @@ while :; do
__Keyring="--keyring $__KeyringFile"
fi
;;
+ loongarch64)
+ __BuildArch=loongarch64
+ __AlpineArch=loongarch64
+ __QEMUArch=loongarch64
+ __UbuntuArch=loong64
+ __UbuntuSuites=unreleased
+ __LLDB_Package="liblldb-19-dev"
+
+ if [[ "$__CodeName" == "sid" ]]; then
+ __UbuntuRepo="http://ftp.ports.debian.org/debian-ports/"
+ fi
+ ;;
riscv64)
__BuildArch=riscv64
__AlpineArch=riscv64
@@ -264,44 +282,21 @@ while :; do
;;
xenial) # Ubuntu 16.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=xenial
- fi
- ;;
- zesty) # Ubuntu 17.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=zesty
- fi
+ __CodeName=xenial
;;
bionic) # Ubuntu 18.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=bionic
- fi
+ __CodeName=bionic
;;
focal) # Ubuntu 20.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=focal
- fi
+ __CodeName=focal
;;
jammy) # Ubuntu 22.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=jammy
- fi
+ __CodeName=jammy
;;
noble) # Ubuntu 24.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=noble
- fi
- if [[ -n "$__LLDB_Package" ]]; then
- __LLDB_Package="liblldb-18-dev"
- fi
- ;;
- jessie) # Debian 8
- __CodeName=jessie
- __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
-
- if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ftp.debian.org/debian/"
+ __CodeName=noble
+ if [[ -z "$__LLDB_Package" ]]; then
+ __LLDB_Package="liblldb-19-dev"
fi
;;
stretch) # Debian 9
@@ -319,7 +314,7 @@ while :; do
__KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ftp.debian.org/debian/"
+ __UbuntuRepo="http://archive.debian.org/debian/"
fi
;;
bullseye) # Debian 11
@@ -340,10 +335,28 @@ while :; do
;;
sid) # Debian sid
__CodeName=sid
- __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
+ __UbuntuSuites=
- if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ftp.debian.org/debian/"
+ # Debian-Ports architectures need different values
+ case "$__UbuntuArch" in
+ amd64|arm64|armel|armhf|i386|mips64el|ppc64el|riscv64|s390x)
+ __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
+
+ if [[ -z "$__UbuntuRepo" ]]; then
+ __UbuntuRepo="http://ftp.debian.org/debian/"
+ fi
+ ;;
+ *)
+ __KeyringFile="/usr/share/keyrings/debian-ports-archive-keyring.gpg"
+
+ if [[ -z "$__UbuntuRepo" ]]; then
+ __UbuntuRepo="http://ftp.ports.debian.org/debian-ports/"
+ fi
+ ;;
+ esac
+
+ if [[ -e "$__KeyringFile" ]]; then
+ __Keyring="--keyring $__KeyringFile"
fi
;;
tizen)
@@ -370,7 +383,7 @@ while :; do
;;
freebsd14)
__CodeName=freebsd
- __FreeBSDBase="14.0-RELEASE"
+ __FreeBSDBase="14.2-RELEASE"
__FreeBSDABI="14"
__SkipUnmount=1
;;
@@ -388,6 +401,9 @@ while :; do
--skipsigcheck)
__SkipSigCheck=1
;;
+ --skipemulation)
+ __SkipEmulation=1
+ ;;
--rootfsdir|-rootfsdir)
shift
__RootfsDir="$1"
@@ -420,16 +436,15 @@ case "$__AlpineVersion" in
elif [[ "$__AlpineArch" == "x86" ]]; then
__AlpineVersion=3.17 # minimum version that supports lldb-dev
__AlpinePackages+=" llvm15-libs"
- elif [[ "$__AlpineArch" == "riscv64" ]]; then
+ elif [[ "$__AlpineArch" == "riscv64" || "$__AlpineArch" == "loongarch64" ]]; then
+ __AlpineVersion=3.21 # minimum version that supports lldb-dev
+ __AlpinePackages+=" llvm19-libs"
+ elif [[ -n "$__AlpineMajorVersion" ]]; then
+ # use whichever alpine version is provided and select the latest toolchain libs
__AlpineLlvmLibsLookup=1
- __AlpineVersion=edge # minimum version with APKINDEX.tar.gz (packages archive)
else
__AlpineVersion=3.13 # 3.13 to maximize compatibility
__AlpinePackages+=" llvm10-libs"
-
- if [[ "$__AlpineArch" == "armv7" ]]; then
- __AlpinePackages="${__AlpinePackages//numactl-dev/}"
- fi
fi
esac
@@ -439,15 +454,6 @@ if [[ "$__AlpineVersion" =~ 3\.1[345] ]]; then
__AlpinePackages="${__AlpinePackages/compiler-rt/compiler-rt-static}"
fi
-if [[ "$__BuildArch" == "armel" ]]; then
- __LLDB_Package="lldb-3.5-dev"
-fi
-
-if [[ "$__CodeName" == "xenial" && "$__UbuntuArch" == "armhf" ]]; then
- # libnuma-dev is not available on armhf for xenial
- __UbuntuPackages="${__UbuntuPackages//libnuma-dev/}"
-fi
-
__UbuntuPackages+=" ${__LLDB_Package:-}"
if [[ -z "$__UbuntuRepo" ]]; then
@@ -496,7 +502,7 @@ if [[ "$__CodeName" == "alpine" ]]; then
arch="$(uname -m)"
ensureDownloadTool
-
+
if [[ "$__hasWget" == 1 ]]; then
wget -P "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static"
else
@@ -512,11 +518,6 @@ if [[ "$__CodeName" == "alpine" ]]; then
echo "$__ApkToolsSHA512SUM $__ApkToolsDir/apk.static" | sha512sum -c
chmod +x "$__ApkToolsDir/apk.static"
- if [[ -f "/usr/bin/qemu-$__QEMUArch-static" ]]; then
- mkdir -p "$__RootfsDir"/usr/bin
- cp -v "/usr/bin/qemu-$__QEMUArch-static" "$__RootfsDir/usr/bin"
- fi
-
if [[ "$__AlpineVersion" == "edge" ]]; then
version=edge
else
@@ -536,6 +537,10 @@ if [[ "$__CodeName" == "alpine" ]]; then
__ApkSignatureArg="--keys-dir $__ApkKeysDir"
fi
+ if [[ "$__SkipEmulation" == "1" ]]; then
+ __NoEmulationArg="--no-scripts"
+ fi
+
# initialize DB
# shellcheck disable=SC2086
"$__ApkToolsDir/apk.static" \
@@ -557,7 +562,7 @@ if [[ "$__CodeName" == "alpine" ]]; then
"$__ApkToolsDir/apk.static" \
-X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \
-X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \
- -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \
+ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" $__NoEmulationArg \
add $__AlpinePackages
rm -r "$__ApkToolsDir"
@@ -573,7 +578,7 @@ elif [[ "$__CodeName" == "freebsd" ]]; then
curl -SL "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
fi
echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf
- echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf
+ echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf
mkdir -p "$__RootfsDir"/tmp
# get and build package manager
if [[ "$__hasWget" == 1 ]]; then
@@ -681,7 +686,7 @@ elif [[ "$__CodeName" == "haiku" ]]; then
ensureDownloadTool
- echo "Downloading Haiku package tool"
+ echo "Downloading Haiku package tools"
git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 "$__RootfsDir/tmp/script"
if [[ "$__hasWget" == 1 ]]; then
wget -O "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)"
@@ -691,34 +696,42 @@ elif [[ "$__CodeName" == "haiku" ]]; then
unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin"
- DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg"
- HpkgBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current"
+ HaikuBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current"
+ HaikuPortsBaseUrl="https://eu.hpkg.haiku-os.org/haikuports/master/$__HaikuArch/current"
+
+ echo "Downloading HaikuPorts package repository index..."
+ if [[ "$__hasWget" == 1 ]]; then
+ wget -P "$__RootfsDir/tmp/download" "$HaikuPortsBaseUrl/repo"
+ else
+ curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HaikuPortsBaseUrl/repo"
+ fi
- # Download Haiku packages
echo "Downloading Haiku packages"
read -ra array <<<"$__HaikuPackages"
for package in "${array[@]}"; do
echo "Downloading $package..."
- # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60
- # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598
+ hpkgFilename="$(LD_LIBRARY_PATH="$__RootfsDir/tmp/bin" "$__RootfsDir/tmp/bin/package_repo" list -f "$__RootfsDir/tmp/download/repo" |
+ grep -E "${package}-" | sort -V | tail -n 1 | xargs)"
+ if [ -z "$hpkgFilename" ]; then
+ >&2 echo "ERROR: package $package missing."
+ exit 1
+ fi
+ echo "Resolved filename: $hpkgFilename..."
+ hpkgDownloadUrl="$HaikuPortsBaseUrl/packages/$hpkgFilename"
if [[ "$__hasWget" == 1 ]]; then
- hpkgDownloadUrl="$(wget -qO- --post-data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
- --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
else
- hpkgDownloadUrl="$(curl -sSL -XPOST --data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
- --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
fi
done
for package in haiku haiku_devel; do
echo "Downloading $package..."
if [[ "$__hasWget" == 1 ]]; then
- hpkgVersion="$(wget -qO- "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
- wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
+ hpkgVersion="$(wget -qO- "$HaikuBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
+ wget -P "$__RootfsDir/tmp/download" "$HaikuBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
else
- hpkgVersion="$(curl -sSL "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
- curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
+ hpkgVersion="$(curl -sSL "$HaikuBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
+ curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HaikuBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
fi
done
@@ -744,25 +757,67 @@ elif [[ "$__CodeName" == "haiku" ]]; then
popd
rm -rf "$__RootfsDir/tmp"
elif [[ -n "$__CodeName" ]]; then
+ __Suites="$__CodeName $(for suite in $__UbuntuSuites; do echo -n "$__CodeName-$suite "; done)"
+
+ if [[ "$__SkipEmulation" == "1" ]]; then
+ if [[ -z "$AR" ]]; then
+ if command -v ar &>/dev/null; then
+ AR="$(command -v ar)"
+ elif command -v llvm-ar &>/dev/null; then
+ AR="$(command -v llvm-ar)"
+ else
+ echo "Unable to find ar or llvm-ar on PATH, add them to PATH or set AR environment variable pointing to the available AR tool"
+ exit 1
+ fi
+ fi
+
+ PYTHON=${PYTHON_EXECUTABLE:-python3}
+
+ # shellcheck disable=SC2086,SC2046
+ echo running "$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \
+ $(for suite in $__Suites; do echo -n "--suite $suite "; done) \
+ $__UbuntuPackages
+
+ # shellcheck disable=SC2086,SC2046
+ "$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \
+ $(for suite in $__Suites; do echo -n "--suite $suite "; done) \
+ $__UbuntuPackages
+ exit 0
+ fi
+
+ __UpdateOptions=
if [[ "$__SkipSigCheck" == "0" ]]; then
__Keyring="$__Keyring --force-check-gpg"
+ else
+ __Keyring=
+ __UpdateOptions="--allow-unauthenticated --allow-insecure-repositories"
fi
# shellcheck disable=SC2086
echo running debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"
- debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"
+ # shellcheck disable=SC2086
+ if ! debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"; then
+ echo "debootstrap failed! dumping debootstrap.log"
+ cat "$__RootfsDir/debootstrap/debootstrap.log"
+ exit 1
+ fi
+
+ rm -rf "$__RootfsDir"/etc/apt/*.{sources,list} "$__RootfsDir"/etc/apt/sources.list.d
mkdir -p "$__RootfsDir/etc/apt/sources.list.d/"
+
+ # shellcheck disable=SC2086
cat > "$__RootfsDir/etc/apt/sources.list.d/$__CodeName.sources" < token2) - (token1 < token2)
+ else:
+ return -1 if isinstance(token1, str) else 1
+
+ return len(tokens1) - len(tokens2)
+
+def compare_debian_versions(version1, version2):
+ """Compare two Debian package versions."""
+ epoch1, upstream1, revision1 = parse_debian_version(version1)
+ epoch2, upstream2, revision2 = parse_debian_version(version2)
+
+ if epoch1 != epoch2:
+ return epoch1 - epoch2
+
+ result = compare_upstream_version(upstream1, upstream2)
+ if result != 0:
+ return result
+
+ return compare_upstream_version(revision1, revision2)
+
+def resolve_dependencies(packages, aliases, desired_packages):
+ """Recursively resolves dependencies for the desired packages."""
+ resolved = []
+ to_process = deque(desired_packages)
+
+ while to_process:
+ current = to_process.popleft()
+ resolved_package = current if current in packages else aliases.get(current, [None])[0]
+
+ if not resolved_package:
+ print(f"Error: Package '{current}' was not found in the available packages.")
+ sys.exit(1)
+
+ if resolved_package not in resolved:
+ resolved.append(resolved_package)
+
+ deps = packages.get(resolved_package, {}).get("Depends", "")
+ if deps:
+ deps = [dep.split(' ')[0] for dep in deps.split(', ') if dep]
+ for dep in deps:
+ if dep not in resolved and dep not in to_process and dep in packages:
+ to_process.append(dep)
+
+ return resolved
+
+def parse_package_index(content):
+ """Parses the Packages.gz file and returns package information."""
+ packages = {}
+ aliases = {}
+ entries = re.split(r'\n\n+', content)
+
+ for entry in entries:
+ fields = dict(re.findall(r'^(\S+): (.+)$', entry, re.MULTILINE))
+ if "Package" in fields:
+ package_name = fields["Package"]
+ version = fields.get("Version")
+ filename = fields.get("Filename")
+ depends = fields.get("Depends")
+ provides = fields.get("Provides", None)
+
+ # Only update if package_name is not in packages or if the new version is higher
+ if package_name not in packages or compare_debian_versions(version, packages[package_name]["Version"]) > 0:
+ packages[package_name] = {
+ "Version": version,
+ "Filename": filename,
+ "Depends": depends
+ }
+
+ # Update aliases if package provides any alternatives
+ if provides:
+ provides_list = [x.strip() for x in provides.split(",")]
+ for alias in provides_list:
+ # Strip version specifiers
+ alias_name = re.sub(r'\s*\(=.*\)', '', alias)
+ if alias_name not in aliases:
+ aliases[alias_name] = []
+ if package_name not in aliases[alias_name]:
+ aliases[alias_name].append(package_name)
+
+ return packages, aliases
+
+def install_packages(mirror, packages_info, aliases, tmp_dir, extract_dir, ar_tool, desired_packages):
+ """Downloads .deb files and extracts them."""
+ resolved_packages = resolve_dependencies(packages_info, aliases, desired_packages)
+ print(f"Resolved packages (including dependencies): {resolved_packages}")
+
+ packages_to_download = {}
+
+ for pkg in resolved_packages:
+ if pkg in packages_info:
+ packages_to_download[pkg] = packages_info[pkg]
+
+ if pkg in aliases:
+ for alias in aliases[pkg]:
+ if alias in packages_info:
+ packages_to_download[alias] = packages_info[alias]
+
+ asyncio.run(download_deb_files_parallel(mirror, packages_to_download, tmp_dir))
+
+ package_to_deb_file_map = {}
+ for pkg in resolved_packages:
+ pkg_info = packages_info.get(pkg)
+ if pkg_info:
+ deb_filename = pkg_info.get("Filename")
+ if deb_filename:
+ deb_file_path = os.path.join(tmp_dir, os.path.basename(deb_filename))
+ package_to_deb_file_map[pkg] = deb_file_path
+
+ for pkg in reversed(resolved_packages):
+ deb_file = package_to_deb_file_map.get(pkg)
+ if deb_file and os.path.exists(deb_file):
+ extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool)
+
+ print("All done!")
+
+def extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool):
+ """Extract .deb file contents"""
+
+ os.makedirs(extract_dir, exist_ok=True)
+
+ with tempfile.TemporaryDirectory(dir=tmp_dir) as tmp_subdir:
+ result = subprocess.run(f"{ar_tool} t {os.path.abspath(deb_file)}", cwd=tmp_subdir, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+ tar_filename = None
+ for line in result.stdout.decode().splitlines():
+ if line.startswith("data.tar"):
+ tar_filename = line.strip()
+ break
+
+ if not tar_filename:
+ raise FileNotFoundError(f"Could not find 'data.tar.*' in {deb_file}.")
+
+ tar_file_path = os.path.join(tmp_subdir, tar_filename)
+ print(f"Extracting {tar_filename} from {deb_file}..")
+
+ subprocess.run(f"{ar_tool} p {os.path.abspath(deb_file)} {tar_filename} > {tar_file_path}", check=True, shell=True)
+
+ file_extension = os.path.splitext(tar_file_path)[1].lower()
+
+ if file_extension == ".xz":
+ mode = "r:xz"
+ elif file_extension == ".gz":
+ mode = "r:gz"
+ elif file_extension == ".zst":
+ # zstd is not supported by standard library yet
+ decompressed_tar_path = tar_file_path.replace(".zst", "")
+ with open(tar_file_path, "rb") as zst_file, open(decompressed_tar_path, "wb") as decompressed_file:
+ dctx = zstandard.ZstdDecompressor()
+ dctx.copy_stream(zst_file, decompressed_file)
+
+ tar_file_path = decompressed_tar_path
+ mode = "r"
+ else:
+ raise ValueError(f"Unsupported compression format: {file_extension}")
+
+ with tarfile.open(tar_file_path, mode) as tar:
+ tar.extractall(path=extract_dir, filter='fully_trusted')
+
+def finalize_setup(rootfsdir):
+ lib_dir = os.path.join(rootfsdir, 'lib')
+ usr_lib_dir = os.path.join(rootfsdir, 'usr', 'lib')
+
+ if os.path.exists(lib_dir):
+ if os.path.islink(lib_dir):
+ os.remove(lib_dir)
+ else:
+ os.makedirs(usr_lib_dir, exist_ok=True)
+
+ for item in os.listdir(lib_dir):
+ src = os.path.join(lib_dir, item)
+ dest = os.path.join(usr_lib_dir, item)
+
+ if os.path.isdir(src):
+ shutil.copytree(src, dest, dirs_exist_ok=True)
+ else:
+ shutil.copy2(src, dest)
+
+ shutil.rmtree(lib_dir)
+
+ os.symlink(usr_lib_dir, lib_dir)
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Generate rootfs for .NET runtime on Debian-like OS")
+ parser.add_argument("--distro", required=False, help="Distro name (e.g., debian, ubuntu, etc.)")
+ parser.add_argument("--arch", required=True, help="Architecture (e.g., amd64, loong64, etc.)")
+ parser.add_argument("--rootfsdir", required=True, help="Destination directory.")
+ parser.add_argument('--suite', required=True, action='append', help='Specify one or more repository suites to collect index data.')
+ parser.add_argument("--mirror", required=False, help="Mirror (e.g., http://ftp.debian.org/debian-ports etc.)")
+ parser.add_argument("--artool", required=False, default="ar", help="ar tool to extract debs (e.g., ar, llvm-ar etc.)")
+ parser.add_argument("packages", nargs="+", help="List of package names to be installed.")
+
+ args = parser.parse_args()
+
+ if args.mirror is None:
+ if args.distro == "ubuntu":
+ args.mirror = "http://archive.ubuntu.com/ubuntu" if args.arch in ["amd64", "i386"] else "http://ports.ubuntu.com/ubuntu-ports"
+ elif args.distro == "debian":
+ args.mirror = "http://ftp.debian.org/debian-ports"
+ else:
+ raise Exception("Unsupported distro")
+
+ DESIRED_PACKAGES = args.packages + [ # base packages
+ "dpkg",
+ "busybox",
+ "libc-bin",
+ "base-files",
+ "base-passwd",
+ "debianutils"
+ ]
+
+ print(f"Creating rootfs. rootfsdir: {args.rootfsdir}, distro: {args.distro}, arch: {args.arch}, suites: {args.suite}, mirror: {args.mirror}")
+
+ package_index_content = asyncio.run(download_package_index_parallel(args.mirror, args.arch, args.suite))
+
+ packages_info, aliases = parse_package_index(package_index_content)
+
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ install_packages(args.mirror, packages_info, aliases, tmp_dir, args.rootfsdir, args.artool, DESIRED_PACKAGES)
+
+ finalize_setup(args.rootfsdir)
diff --git a/eng/common/cross/tizen-fetch.sh b/eng/common/cross/tizen-fetch.sh
index 28936cee..37c3a61f 100755
--- a/eng/common/cross/tizen-fetch.sh
+++ b/eng/common/cross/tizen-fetch.sh
@@ -156,13 +156,8 @@ fetch_tizen_pkgs()
done
}
-if [ "$TIZEN_ARCH" == "riscv64" ]; then
- BASE="Tizen-Base-RISCV"
- UNIFIED="Tizen-Unified-RISCV"
-else
- BASE="Tizen-Base"
- UNIFIED="Tizen-Unified"
-fi
+BASE="Tizen-Base"
+UNIFIED="Tizen-Unified"
Inform "Initialize ${TIZEN_ARCH} base"
fetch_tizen_pkgs_init standard $BASE
diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake
index 9a4e285a..0ff85cf0 100644
--- a/eng/common/cross/toolchain.cmake
+++ b/eng/common/cross/toolchain.cmake
@@ -40,7 +40,7 @@ if(TARGET_ARCH_NAME STREQUAL "arm")
set(TOOLCHAIN "arm-linux-gnueabihf")
endif()
if(TIZEN)
- set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0")
+ set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf")
endif()
elseif(TARGET_ARCH_NAME STREQUAL "arm64")
set(CMAKE_SYSTEM_PROCESSOR aarch64)
@@ -49,7 +49,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64")
elseif(LINUX)
set(TOOLCHAIN "aarch64-linux-gnu")
if(TIZEN)
- set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0")
+ set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu")
endif()
elseif(FREEBSD)
set(triple "aarch64-unknown-freebsd12")
@@ -58,7 +58,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "armel")
set(CMAKE_SYSTEM_PROCESSOR armv7l)
set(TOOLCHAIN "arm-linux-gnueabi")
if(TIZEN)
- set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0")
+ set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi")
endif()
elseif(TARGET_ARCH_NAME STREQUAL "armv6")
set(CMAKE_SYSTEM_PROCESSOR armv6l)
@@ -67,6 +67,13 @@ elseif(TARGET_ARCH_NAME STREQUAL "armv6")
else()
set(TOOLCHAIN "arm-linux-gnueabihf")
endif()
+elseif(TARGET_ARCH_NAME STREQUAL "loongarch64")
+ set(CMAKE_SYSTEM_PROCESSOR "loongarch64")
+ if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/loongarch64-alpine-linux-musl)
+ set(TOOLCHAIN "loongarch64-alpine-linux-musl")
+ else()
+ set(TOOLCHAIN "loongarch64-linux-gnu")
+ endif()
elseif(TARGET_ARCH_NAME STREQUAL "ppc64le")
set(CMAKE_SYSTEM_PROCESSOR ppc64le)
if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/powerpc64le-alpine-linux-musl)
@@ -81,7 +88,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "riscv64")
else()
set(TOOLCHAIN "riscv64-linux-gnu")
if(TIZEN)
- set(TIZEN_TOOLCHAIN "riscv64-tizen-linux-gnu/13.1.0")
+ set(TIZEN_TOOLCHAIN "riscv64-tizen-linux-gnu")
endif()
endif()
elseif(TARGET_ARCH_NAME STREQUAL "s390x")
@@ -98,7 +105,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64")
elseif(LINUX)
set(TOOLCHAIN "x86_64-linux-gnu")
if(TIZEN)
- set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/9.2.0")
+ set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu")
endif()
elseif(FREEBSD)
set(triple "x86_64-unknown-freebsd12")
@@ -115,10 +122,10 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86")
set(TOOLCHAIN "i686-linux-gnu")
endif()
if(TIZEN)
- set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0")
+ set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu")
endif()
else()
- message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, ppc64le, riscv64, s390x, x64 and x86 are supported!")
+ message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, loongarch64, ppc64le, riscv64, s390x, x64 and x86 are supported!")
endif()
if(DEFINED ENV{TOOLCHAIN})
@@ -127,32 +134,46 @@ endif()
# Specify include paths
if(TIZEN)
- if(TARGET_ARCH_NAME STREQUAL "arm")
- include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
- include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7hl-tizen-linux-gnueabihf)
- endif()
- if(TARGET_ARCH_NAME STREQUAL "armel")
- include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
- include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi)
- endif()
- if(TARGET_ARCH_NAME STREQUAL "arm64")
- include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
- include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/aarch64-tizen-linux-gnu)
- endif()
- if(TARGET_ARCH_NAME STREQUAL "x86")
- include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
- include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/i586-tizen-linux-gnu)
- endif()
- if(TARGET_ARCH_NAME STREQUAL "x64")
- include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
- include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/x86_64-tizen-linux-gnu)
- endif()
- if(TARGET_ARCH_NAME STREQUAL "riscv64")
- include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
- include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/riscv64-tizen-linux-gnu)
+ function(find_toolchain_dir prefix)
+ # Dynamically find the version subdirectory
+ file(GLOB DIRECTORIES "${prefix}/*")
+ list(GET DIRECTORIES 0 FIRST_MATCH)
+ get_filename_component(TOOLCHAIN_VERSION ${FIRST_MATCH} NAME)
+
+ set(TIZEN_TOOLCHAIN_PATH "${prefix}/${TOOLCHAIN_VERSION}" PARENT_SCOPE)
+ endfunction()
+
+ if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$")
+ find_toolchain_dir("${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+ else()
+ find_toolchain_dir("${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
endif()
+
+ message(STATUS "TIZEN_TOOLCHAIN_PATH set to: ${TIZEN_TOOLCHAIN_PATH}")
+
+ include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++)
+ include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++/${TIZEN_TOOLCHAIN})
endif()
+function(locate_toolchain_exec exec var)
+ set(TOOLSET_PREFIX ${TOOLCHAIN}-)
+ string(TOUPPER ${exec} EXEC_UPPERCASE)
+ if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "")
+ set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE)
+ return()
+ endif()
+
+ find_program(EXEC_LOCATION_${exec}
+ NAMES
+ "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
+ "${TOOLSET_PREFIX}${exec}")
+
+ if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND")
+ message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.")
+ endif()
+ set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE)
+endfunction()
+
if(ANDROID)
if(TARGET_ARCH_NAME STREQUAL "arm")
set(ANDROID_ABI armeabi-v7a)
@@ -183,66 +204,24 @@ elseif(FREEBSD)
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld")
elseif(ILLUMOS)
set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
+ set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}")
+ set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
+ set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
include_directories(SYSTEM ${CROSS_ROOTFS}/include)
- set(TOOLSET_PREFIX ${TOOLCHAIN}-)
- function(locate_toolchain_exec exec var)
- string(TOUPPER ${exec} EXEC_UPPERCASE)
- if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "")
- set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE)
- return()
- endif()
-
- find_program(EXEC_LOCATION_${exec}
- NAMES
- "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
- "${TOOLSET_PREFIX}${exec}")
-
- if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND")
- message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.")
- endif()
- set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE)
- endfunction()
-
- set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}")
-
locate_toolchain_exec(gcc CMAKE_C_COMPILER)
locate_toolchain_exec(g++ CMAKE_CXX_COMPILER)
-
- set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
- set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
elseif(HAIKU)
set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin")
-
- set(TOOLSET_PREFIX ${TOOLCHAIN}-)
- function(locate_toolchain_exec exec var)
- string(TOUPPER ${exec} EXEC_UPPERCASE)
- if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "")
- set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE)
- return()
- endif()
-
- find_program(EXEC_LOCATION_${exec}
- NAMES
- "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
- "${TOOLSET_PREFIX}${exec}")
-
- if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND")
- message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.")
- endif()
- set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE)
- endfunction()
-
set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}")
+ set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
+ set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
locate_toolchain_exec(gcc CMAKE_C_COMPILER)
locate_toolchain_exec(g++ CMAKE_CXX_COMPILER)
- set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
- set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
-
# let CMake set up the correct search paths
include(Platform/Haiku)
else()
@@ -272,21 +251,21 @@ endif()
if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
if(TIZEN)
- add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+ add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}")
add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib")
add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib")
- add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+ add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}")
endif()
elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64|riscv64)$")
if(TIZEN)
- add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
+ add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}")
add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64")
add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64")
- add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
+ add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}")
add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib64")
add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64")
- add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
+ add_toolchain_linker_flag("-Wl,--rpath-link=${TIZEN_TOOLCHAIN_PATH}")
endif()
elseif(TARGET_ARCH_NAME STREQUAL "s390x")
add_toolchain_linker_flag("--target=${TOOLCHAIN}")
@@ -297,10 +276,10 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86")
endif()
add_toolchain_linker_flag(-m32)
if(TIZEN)
- add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+ add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}")
add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib")
add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib")
- add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+ add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}")
endif()
elseif(ILLUMOS)
add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64")
@@ -312,7 +291,7 @@ endif()
# Specify compile options
-if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU)
+if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|loongarch64|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU)
set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN})
diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh
index 36dbd45e..e889f439 100755
--- a/eng/common/darc-init.sh
+++ b/eng/common/darc-init.sh
@@ -68,7 +68,7 @@ function InstallDarcCli {
fi
fi
- local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json"
+ local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json"
echo "Installing Darc CLI version $darcVersion..."
echo "You may need to restart your command shell if this is the first dotnet tool you have installed."
diff --git a/eng/common/dotnet.cmd b/eng/common/dotnet.cmd
new file mode 100644
index 00000000..527fa4bb
--- /dev/null
+++ b/eng/common/dotnet.cmd
@@ -0,0 +1,7 @@
+@echo off
+
+:: This script is used to install the .NET SDK.
+:: It will also invoke the SDK with any provided arguments.
+
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet.ps1""" %*"
+exit /b %ErrorLevel%
diff --git a/eng/common/dotnet.ps1 b/eng/common/dotnet.ps1
new file mode 100644
index 00000000..45e5676c
--- /dev/null
+++ b/eng/common/dotnet.ps1
@@ -0,0 +1,11 @@
+# This script is used to install the .NET SDK.
+# It will also invoke the SDK with any provided arguments.
+
+. $PSScriptRoot\tools.ps1
+$dotnetRoot = InitializeDotNetCli -install:$true
+
+# Invoke acquired SDK with args if they are provided
+if ($args.count -gt 0) {
+ $env:DOTNET_NOLOGO=1
+ & "$dotnetRoot\dotnet.exe" $args
+}
diff --git a/eng/common/dotnet.sh b/eng/common/dotnet.sh
new file mode 100755
index 00000000..2ef68235
--- /dev/null
+++ b/eng/common/dotnet.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# This script is used to install the .NET SDK.
+# It will also invoke the SDK with any provided arguments.
+
+source="${BASH_SOURCE[0]}"
+# resolve $SOURCE until the file is no longer a symlink
+while [[ -h $source ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+source $scriptroot/tools.sh
+InitializeDotNetCli true # install
+
+# Invoke acquired SDK with args if they are provided
+if [[ $# > 0 ]]; then
+ __dotnetDir=${_InitializeDotNetCli}
+ dotnetPath=${__dotnetDir}/dotnet
+ ${dotnetPath} "$@"
+fi
diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1
index 524aaa57..fa1cdc2b 100644
--- a/eng/common/generate-locproject.ps1
+++ b/eng/common/generate-locproject.ps1
@@ -33,15 +33,27 @@ $jsonTemplateFiles | ForEach-Object {
$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
+$wxlFilesV3 = @()
+$wxlFilesV5 = @()
$wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them
if (-not $wxlFiles) {
$wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files
if ($wxlEnFiles) {
- $wxlFiles = @()
- $wxlEnFiles | ForEach-Object {
- $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)"
- $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
- }
+ $wxlFiles = @()
+ $wxlEnFiles | ForEach-Object {
+ $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)"
+ $content = Get-Content $_.FullName -Raw
+
+ # Split files on schema to select different parser settings in the generated project.
+ if ($content -like "*http://wixtoolset.org/schemas/v4/wxl*")
+ {
+ $wxlFilesV5 += Copy-Item $_.FullName -Destination $destinationFile -PassThru
+ }
+ elseif ($content -like "*http://schemas.microsoft.com/wix/2006/localization*")
+ {
+ $wxlFilesV3 += Copy-Item $_.FullName -Destination $destinationFile -PassThru
+ }
+ }
}
}
@@ -114,7 +126,32 @@ $locJson = @{
CloneLanguageSet = "WiX_CloneLanguages"
LssFiles = @( "wxl_loc.lss" )
LocItems = @(
- $wxlFiles | ForEach-Object {
+ $wxlFilesV3 | ForEach-Object {
+ $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
+ $continue = $true
+ foreach ($exclusion in $exclusions.Exclusions) {
+ if ($_.FullName.Contains($exclusion)) {
+ $continue = $false
+ }
+ }
+ $sourceFile = ($_.FullName | Resolve-Path -Relative)
+ if ($continue)
+ {
+ return @{
+ SourceFile = $sourceFile
+ CopyOption = "LangIDOnPath"
+ OutputPath = $outputPath
+ }
+ }
+ }
+ )
+ },
+ @{
+ LanguageSet = $LanguageSet
+ CloneLanguageSet = "WiX_CloneLanguages"
+ LssFiles = @( "P210WxlSchemaV4.lss" )
+ LocItems = @(
+ $wxlFilesV5 | ForEach-Object {
$outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
$continue = $true
foreach ($exclusion in $exclusions.Exclusions) {
diff --git a/eng/common/generate-sbom-prep.ps1 b/eng/common/generate-sbom-prep.ps1
index 3e5c1c74..a0c7d792 100644
--- a/eng/common/generate-sbom-prep.ps1
+++ b/eng/common/generate-sbom-prep.ps1
@@ -4,18 +4,26 @@ Param(
. $PSScriptRoot\pipeline-logging-functions.ps1
+# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly
+# with their own overwriting ours. So we create it as a sub directory of the requested manifest path.
+$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM"
+$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_'
+$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName
+
+Write-Host "Artifact name before : $ArtifactName"
+Write-Host "Artifact name after : $SafeArtifactName"
+
Write-Host "Creating dir $ManifestDirPath"
+
# create directory for sbom manifest to be placed
-if (!(Test-Path -path $ManifestDirPath))
+if (!(Test-Path -path $SbomGenerationDir))
{
- New-Item -ItemType Directory -path $ManifestDirPath
- Write-Host "Successfully created directory $ManifestDirPath"
+ New-Item -ItemType Directory -path $SbomGenerationDir
+ Write-Host "Successfully created directory $SbomGenerationDir"
}
else{
Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder."
}
Write-Host "Updating artifact name"
-$artifact_name = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -replace '["/:<>\\|?@*"() ]', '_'
-Write-Host "Artifact name $artifact_name"
-Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$artifact_name"
+Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName"
diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh
index d5c76dc8..b8ecca72 100755
--- a/eng/common/generate-sbom-prep.sh
+++ b/eng/common/generate-sbom-prep.sh
@@ -14,19 +14,24 @@ done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
. $scriptroot/pipeline-logging-functions.sh
+
+# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts.
+artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM"
+safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}"
manifest_dir=$1
-if [ ! -d "$manifest_dir" ] ; then
- mkdir -p "$manifest_dir"
- echo "Sbom directory created." $manifest_dir
+# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly
+# with their own overwriting ours. So we create it as a sub directory of the requested manifest path.
+sbom_generation_dir="$manifest_dir/$safe_artifact_name"
+
+if [ ! -d "$sbom_generation_dir" ] ; then
+ mkdir -p "$sbom_generation_dir"
+ echo "Sbom directory created." $sbom_generation_dir
else
Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder."
fi
-artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM"
echo "Artifact name before : "$artifact_name
-# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts.
-safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}"
echo "Artifact name after : "$safe_artifact_name
export ARTIFACT_NAME=$safe_artifact_name
echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name"
diff --git a/eng/common/internal/NuGet.config b/eng/common/internal/NuGet.config
index 19d3d311..f70261ed 100644
--- a/eng/common/internal/NuGet.config
+++ b/eng/common/internal/NuGet.config
@@ -4,4 +4,7 @@
+
+
+
diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj
index 32f79dfb..feaa6d20 100644
--- a/eng/common/internal/Tools.csproj
+++ b/eng/common/internal/Tools.csproj
@@ -15,16 +15,6 @@
-
-
-
- https://devdiv.pkgs.visualstudio.com/_packaging/dotnet-core-internal-tooling/nuget/v3/index.json;
-
-
- $(RestoreSources);
- https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json;
-
-
diff --git a/eng/common/native/install-dependencies.sh b/eng/common/native/install-dependencies.sh
new file mode 100755
index 00000000..477a44f3
--- /dev/null
+++ b/eng/common/native/install-dependencies.sh
@@ -0,0 +1,62 @@
+#!/bin/sh
+
+set -e
+
+# This is a simple script primarily used for CI to install necessary dependencies
+#
+# Usage:
+#
+# ./install-dependencies.sh
+
+os="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
+
+if [ -z "$os" ]; then
+ . "$(dirname "$0")"/init-os-and-arch.sh
+fi
+
+case "$os" in
+ linux)
+ if [ -e /etc/os-release ]; then
+ . /etc/os-release
+ fi
+
+ if [ "$ID" = "debian" ] || [ "$ID_LIKE" = "debian" ]; then
+ apt update
+
+ apt install -y build-essential gettext locales cmake llvm clang lld lldb liblldb-dev libunwind8-dev libicu-dev liblttng-ust-dev \
+ libssl-dev libkrb5-dev pigz cpio
+
+ localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
+ elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ]; then
+ pkg_mgr="$(command -v tdnf 2>/dev/null || command -v dnf)"
+ $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio
+ elif [ "$ID" = "alpine" ]; then
+ apk add build-base cmake bash curl clang llvm-dev lld lldb krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio
+ else
+ echo "Unsupported distro. distro: $ID"
+ exit 1
+ fi
+ ;;
+
+ osx|maccatalyst|ios|iossimulator|tvos|tvossimulator)
+ echo "Installed xcode version: $(xcode-select -p)"
+
+ export HOMEBREW_NO_INSTALL_CLEANUP=1
+ export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
+ # Skip brew update for now, see https://github.com/actions/setup-python/issues/577
+ # brew update --preinstall
+ brew bundle --no-upgrade --file=- < Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
+ Write-Host " -excludeCIBinaryLog When running on CI, allow no binary log (short: -nobl)"
Write-Host ""
Write-Host "Command line arguments not listed above are passed thru to msbuild."
}
@@ -34,10 +37,11 @@ function Print-Usage() {
function Build([string]$target) {
$logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" }
$log = Join-Path $LogDir "$task$logSuffix.binlog"
+ $binaryLogArg = if ($binaryLog) { "/bl:$log" } else { "" }
$outputPath = Join-Path $ToolsetDir "$task\"
MSBuild $taskProject `
- /bl:$log `
+ $binaryLogArg `
/t:$target `
/p:Configuration=$configuration `
/p:RepoRoot=$RepoRoot `
@@ -64,7 +68,7 @@ try {
$GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
}
if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
- $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.10.0-pre.4.0" -MemberType NoteProperty
+ $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.13.0" -MemberType NoteProperty
}
if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
$xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true
diff --git a/eng/common/sdk-task.sh b/eng/common/sdk-task.sh
new file mode 100755
index 00000000..3270f83f
--- /dev/null
+++ b/eng/common/sdk-task.sh
@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+
+show_usage() {
+ echo "Common settings:"
+ echo " --task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)"
+ echo " --restore Restore dependencies"
+ echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]"
+ echo " --help Print help and exit"
+ echo ""
+
+ echo "Advanced settings:"
+ echo " --excludeCIBinarylog Don't output binary log (short: -nobl)"
+ echo " --noWarnAsError Do not warn as error"
+ echo ""
+ echo "Command line arguments not listed above are passed thru to msbuild."
+}
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $source until the file is no longer a symlink
+while [[ -h "$source" ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+Build() {
+ local target=$1
+ local log_suffix=""
+ [[ "$target" != "Execute" ]] && log_suffix=".$target"
+ local log="$log_dir/$task$log_suffix.binlog"
+ local binaryLogArg=""
+ [[ $binary_log == true ]] && binaryLogArg="/bl:$log"
+ local output_path="$toolset_dir/$task/"
+
+ MSBuild "$taskProject" \
+ $binaryLogArg \
+ /t:"$target" \
+ /p:Configuration="$configuration" \
+ /p:RepoRoot="$repo_root" \
+ /p:BaseIntermediateOutputPath="$output_path" \
+ /v:"$verbosity" \
+ $properties
+}
+
+binary_log=true
+configuration="Debug"
+verbosity="minimal"
+exclude_ci_binary_log=false
+restore=false
+help=false
+properties=''
+warnAsError=true
+
+while (($# > 0)); do
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
+ case $lowerI in
+ --task)
+ task=$2
+ shift 2
+ ;;
+ --restore)
+ restore=true
+ shift 1
+ ;;
+ --verbosity)
+ verbosity=$2
+ shift 2
+ ;;
+ --excludecibinarylog|--nobl)
+ binary_log=false
+ exclude_ci_binary_log=true
+ shift 1
+ ;;
+ --noWarnAsError)
+ warnAsError=false
+ shift 1
+ ;;
+ --help)
+ help=true
+ shift 1
+ ;;
+ *)
+ properties="$properties $1"
+ shift 1
+ ;;
+ esac
+done
+
+ci=true
+
+if $help; then
+ show_usage
+ exit 0
+fi
+
+. "$scriptroot/tools.sh"
+InitializeToolset
+
+if [[ -z "$task" ]]; then
+ Write-PipelineTelemetryError -Category 'Task' -Name 'MissingTask' -Message "Missing required parameter '-task '"
+ ExitWithExitCode 1
+fi
+
+taskProject=$(GetSdkTaskProject "$task")
+if [[ ! -e "$taskProject" ]]; then
+ Write-PipelineTelemetryError -Category 'Task' -Name 'UnknownTask' -Message "Unknown task: $task"
+ ExitWithExitCode 1
+fi
+
+if $restore; then
+ Build "Restore"
+fi
+
+Build "Execute"
+
+
+ExitWithExitCode 0
diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config
index 4585cfd6..e5f543ea 100644
--- a/eng/common/sdl/packages.config
+++ b/eng/common/sdl/packages.config
@@ -1,4 +1,4 @@
-
+
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
index 80036474..06b44de7 100644
--- a/eng/common/tools.ps1
+++ b/eng/common/tools.ps1
@@ -42,7 +42,7 @@
[bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true }
# Enable repos to use a particular version of the on-line dotnet-install scripts.
-# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.ps1
+# default URL: https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.ps1
[string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' }
# True to use global NuGet cache instead of restoring packages to repository-local directory.
@@ -65,10 +65,8 @@ $ErrorActionPreference = 'Stop'
# Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed
[string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null }
-# True if the build is a product build
-[bool]$productBuild = if (Test-Path variable:productBuild) { $productBuild } else { $false }
-
-[String[]]$properties = if (Test-Path variable:properties) { $properties } else { @() }
+# True when the build is running within the VMR.
+[bool]$fromVMR = if (Test-Path variable:fromVMR) { $fromVMR } else { $false }
function Create-Directory ([string[]] $path) {
New-Item -Path $path -Force -ItemType 'Directory' | Out-Null
@@ -259,10 +257,23 @@ function Retry($downloadBlock, $maxRetries = 5) {
function GetDotNetInstallScript([string] $dotnetRoot) {
$installScript = Join-Path $dotnetRoot 'dotnet-install.ps1'
+ $shouldDownload = $false
+
if (!(Test-Path $installScript)) {
+ $shouldDownload = $true
+ } else {
+ # Check if the script is older than 30 days
+ $fileAge = (Get-Date) - (Get-Item $installScript).LastWriteTime
+ if ($fileAge.Days -gt 30) {
+ Write-Host "Existing install script is too old, re-downloading..."
+ $shouldDownload = $true
+ }
+ }
+
+ if ($shouldDownload) {
Create-Directory $dotnetRoot
$ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
- $uri = "https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1"
+ $uri = "https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1"
Retry({
Write-Host "GET $uri"
@@ -383,8 +394,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
# If the version of msbuild is going to be xcopied,
# use this version. Version matches a package here:
- # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.10.0-pre.4.0
- $defaultXCopyMSBuildVersion = '17.10.0-pre.4.0'
+ # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.13.0
+ $defaultXCopyMSBuildVersion = '17.13.0'
if (!$vsRequirements) {
if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
@@ -416,7 +427,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
# Locate Visual Studio installation or download x-copy msbuild.
$vsInfo = LocateVisualStudio $vsRequirements
- if ($vsInfo -ne $null) {
+ if ($vsInfo -ne $null -and $env:ForceUseXCopyMSBuild -eq $null) {
# Ensure vsInstallDir has a trailing slash
$vsInstallDir = Join-Path $vsInfo.installationPath "\"
$vsMajorVersion = $vsInfo.installationVersion.Split('.')[0]
@@ -533,7 +544,8 @@ function LocateVisualStudio([object]$vsRequirements = $null){
if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') {
$vswhereVersion = $GlobalJson.tools.vswhere
} else {
- $vswhereVersion = '2.5.2'
+ # keep this in sync with the VSWhereVersion in DefaultVersions.props
+ $vswhereVersion = '3.1.7'
}
$vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion"
@@ -541,7 +553,8 @@ function LocateVisualStudio([object]$vsRequirements = $null){
if (!(Test-Path $vsWhereExe)) {
Create-Directory $vsWhereDir
- Write-Host 'Downloading vswhere'
+ Write-Host "Downloading vswhere $vswhereVersion"
+ $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
Retry({
Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
})
@@ -604,14 +617,7 @@ function InitializeBuildTool() {
}
$dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
- # Use override if it exists - commonly set by source-build
- if ($null -eq $env:_OverrideArcadeInitializeBuildToolFramework) {
- $initializeBuildToolFramework="net9.0"
- } else {
- $initializeBuildToolFramework=$env:_OverrideArcadeInitializeBuildToolFramework
- }
-
- $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = $initializeBuildToolFramework }
+ $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net' }
} elseif ($msbuildEngine -eq "vs") {
try {
$msbuildPath = InitializeVisualStudioMSBuild -install:$restore
@@ -620,7 +626,7 @@ function InitializeBuildTool() {
ExitWithExitCode 1
}
- $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS }
+ $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "netframework"; ExcludePrereleaseVS = $excludePrereleaseVS }
} else {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'."
ExitWithExitCode 1
@@ -653,7 +659,6 @@ function GetNuGetPackageCachePath() {
$env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\'
} else {
$env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\'
- $env:RESTORENOHTTPCACHE = $true
}
}
@@ -775,26 +780,13 @@ function MSBuild() {
$toolsetBuildProject = InitializeToolset
$basePath = Split-Path -parent $toolsetBuildProject
- $possiblePaths = @(
- # new scripts need to work with old packages, so we need to look for the old names/versions
- (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')),
- (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
- (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.ArcadeLogging.dll')),
- (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll')),
- (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.ArcadeLogging.dll')),
- (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.Arcade.Sdk.dll'))
- )
- $selectedPath = $null
- foreach ($path in $possiblePaths) {
- if (Test-Path $path -PathType Leaf) {
- $selectedPath = $path
- break
- }
- }
+ $selectedPath = Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')
+
if (-not $selectedPath) {
- Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.'
+ Write-PipelineTelemetryError -Category 'Build' -Message "Unable to find arcade sdk logger assembly: $selectedPath"
ExitWithExitCode 1
}
+
$args += "/logger:$selectedPath"
}
@@ -857,8 +849,8 @@ function MSBuild-Core() {
}
# When running on Azure Pipelines, override the returned exit code to avoid double logging.
- # Skip this when the build is a child of the VMR orchestrator build.
- if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$productBuild -and -not($properties -like "*DotNetBuildRepo=true*")) {
+ # Skip this when the build is a child of the VMR build.
+ if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$fromVMR) {
Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed."
# Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
# The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
@@ -900,7 +892,7 @@ function IsWindowsPlatform() {
}
function Get-Darc($version) {
- $darcPath = "$TempDir\darc\$(New-Guid)"
+ $darcPath = "$TempDir\darc\$([guid]::NewGuid())"
if ($version -ne $null) {
& $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host
} else {
diff --git a/eng/common/tools.sh b/eng/common/tools.sh
index 1159726a..c1841c9d 100755
--- a/eng/common/tools.sh
+++ b/eng/common/tools.sh
@@ -5,6 +5,9 @@
# CI mode - set to true on CI server for PR validation build or official build.
ci=${ci:-false}
+# Build mode
+source_build=${source_build:-false}
+
# Set to true to use the pipelines logger which will enable Azure logging output.
# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
# This flag is meant as a temporary opt-opt for the feature while validate it across
@@ -54,11 +57,12 @@ warn_as_error=${warn_as_error:-true}
use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
# Enable repos to use a particular version of the on-line dotnet-install scripts.
-# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.sh
+# default URL: https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.sh
dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'}
# True to use global NuGet cache instead of restoring packages to repository-local directory.
-if [[ "$ci" == true ]]; then
+# Keep in sync with NuGetPackageroot in Arcade SDK's RepositoryLayout.props.
+if [[ "$ci" == true || "$source_build" == true ]]; then
use_global_nuget_cache=${use_global_nuget_cache:-false}
else
use_global_nuget_cache=${use_global_nuget_cache:-true}
@@ -68,8 +72,8 @@ fi
runtime_source_feed=${runtime_source_feed:-''}
runtime_source_feed_key=${runtime_source_feed_key:-''}
-# True if the build is a product build
-product_build=${product_build:-false}
+# True when the build is running within the VMR.
+from_vmr=${from_vmr:-false}
# Resolve any symlinks in the given path.
function ResolvePath {
@@ -295,9 +299,30 @@ function with_retries {
function GetDotNetInstallScript {
local root=$1
local install_script="$root/dotnet-install.sh"
- local install_script_url="https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh"
+ local install_script_url="https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh"
+ local timestamp_file="$root/.dotnet-install.timestamp"
+ local should_download=false
if [[ ! -a "$install_script" ]]; then
+ should_download=true
+ elif [[ -f "$timestamp_file" ]]; then
+ # Check if the script is older than 30 days using timestamp file
+ local download_time=$(cat "$timestamp_file" 2>/dev/null || echo "0")
+ local current_time=$(date +%s)
+ local age_seconds=$((current_time - download_time))
+
+ # 30 days = 30 * 24 * 60 * 60 = 2592000 seconds
+ if [[ $age_seconds -gt 2592000 ]]; then
+ echo "Existing install script is too old, re-downloading..."
+ should_download=true
+ fi
+ else
+ # No timestamp file exists, assume script is old and re-download
+ echo "No timestamp found for existing install script, re-downloading..."
+ should_download=true
+ fi
+
+ if [[ "$should_download" == true ]]; then
mkdir -p "$root"
echo "Downloading '$install_script_url'"
@@ -324,6 +349,9 @@ function GetDotNetInstallScript {
ExitWithExitCode $exit_code
}
fi
+
+ # Create timestamp file to track download time in seconds from epoch
+ date +%s > "$timestamp_file"
fi
# return value
_GetDotNetInstallScript="$install_script"
@@ -339,22 +367,14 @@ function InitializeBuildTool {
# return values
_InitializeBuildTool="$_InitializeDotNetCli/dotnet"
_InitializeBuildToolCommand="msbuild"
- # use override if it exists - commonly set by source-build
- if [[ "${_OverrideArcadeInitializeBuildToolFramework:-x}" == "x" ]]; then
- _InitializeBuildToolFramework="net9.0"
- else
- _InitializeBuildToolFramework="${_OverrideArcadeInitializeBuildToolFramework}"
- fi
}
-# Set RestoreNoHttpCache as a workaround for https://github.com/NuGet/Home/issues/3116
function GetNuGetPackageCachePath {
if [[ -z ${NUGET_PACKAGES:-} ]]; then
if [[ "$use_global_nuget_cache" == true ]]; then
export NUGET_PACKAGES="$HOME/.nuget/packages/"
else
export NUGET_PACKAGES="$repo_root/.packages/"
- export RESTORENOHTTPCACHE=true
fi
fi
@@ -451,25 +471,13 @@ function MSBuild {
fi
local toolset_dir="${_InitializeToolset%/*}"
- # new scripts need to work with old packages, so we need to look for the old names/versions
- local selectedPath=
- local possiblePaths=()
- possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" )
- possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" )
- possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.ArcadeLogging.dll" )
- possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.Arcade.Sdk.dll" )
- possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.ArcadeLogging.dll" )
- possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.Arcade.Sdk.dll" )
- for path in "${possiblePaths[@]}"; do
- if [[ -f $path ]]; then
- selectedPath=$path
- break
- fi
- done
+ local selectedPath="$toolset_dir/net/Microsoft.DotNet.ArcadeLogging.dll"
+
if [[ -z "$selectedPath" ]]; then
- Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly."
+ Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly: $selectedPath"
ExitWithExitCode 1
fi
+
args+=( "-logger:$selectedPath" )
fi
@@ -506,8 +514,8 @@ function MSBuild-Core {
echo "Build failed with exit code $exit_code. Check errors above."
# When running on Azure Pipelines, override the returned exit code to avoid double logging.
- # Skip this when the build is a child of the VMR orchestrator build.
- if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$product_build" != true && "$properties" != *"DotNetBuildRepo=true"* ]]; then
+ # Skip this when the build is a child of the VMR build.
+ if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$from_vmr" != true ]]; then
Write-PipelineSetResult -result "Failed" -message "msbuild execution failed."
# Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
# The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
@@ -530,6 +538,13 @@ function GetDarc {
fi
"$eng_root/common/darc-init.sh" --toolpath "$darc_path" $version
+ darc_tool="$darc_path/darc"
+}
+
+# Returns a full path to an Arcade SDK task project file.
+function GetSdkTaskProject {
+ taskName=$1
+ echo "$(dirname $_InitializeToolset)/SdkTasks/$taskName.proj"
}
ResolvePath "${BASH_SOURCE[0]}"
diff --git a/global.json b/global.json
index 764e02e8..b54a37d3 100644
--- a/global.json
+++ b/global.json
@@ -1,16 +1,16 @@
{
"sdk": {
- "version": "9.0.306",
+ "version": "10.0.100-rc.2.25502.107",
"allowPrerelease": true,
"rollForward": "major"
},
"tools": {
- "dotnet": "9.0.306"
+ "dotnet": "10.0.100-rc.2.25502.107"
},
"msbuild-sdks": {
- "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24453.1",
- "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.24453.1"
+ "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25517.104",
+ "Microsoft.DotNet.Helix.Sdk": "10.0.0-beta.25517.104"
}
}
diff --git a/test/AspNet.Security.OpenId.Providers.Tests/Infrastructure/ApplicationFactory.cs b/test/AspNet.Security.OpenId.Providers.Tests/Infrastructure/ApplicationFactory.cs
index 5cfd594b..95a1ea04 100644
--- a/test/AspNet.Security.OpenId.Providers.Tests/Infrastructure/ApplicationFactory.cs
+++ b/test/AspNet.Security.OpenId.Providers.Tests/Infrastructure/ApplicationFactory.cs
@@ -10,6 +10,7 @@
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Http;
using Microsoft.Extensions.Logging;
@@ -60,16 +61,16 @@ private static void Configure(IWebHostBuilder builder, OpenIdTests
{
- // Allow HTTP requests to external services to be intercepted
- services.AddHttpClient();
+ // Allow HTTP requests to external services to be intercepted
+ services.AddHttpClient();
services.AddSingleton(
(_) => new HttpRequestInterceptionFilter(tests.Interceptor));
- // Set up the test endpoint
- services.AddRouting();
+ // Set up the test endpoint
+ services.AddRouting();
- // Configure authentication
- var authentication = services
+ // Configure authentication
+ var authentication = services
.AddAuthentication("External")
.AddCookie("External", o => o.ForwardChallenge = tests.DefaultScheme);
@@ -95,8 +96,8 @@ private static void ConfigureApplication(IApplicationBuilder app)
{
if (context.User.Identity?.IsAuthenticated == true)
{
- var xml = IdentityToXmlString(context.User);
- var buffer = Encoding.UTF8.GetBytes(xml.ToString());
+ string xml = IdentityToXmlString(context.User);
+ byte[] buffer = Encoding.UTF8.GetBytes(xml.ToString());
context.Response.StatusCode = 200;
context.Response.ContentType = "text/xml";
@@ -135,10 +136,15 @@ private static string IdentityToXmlString(ClaimsPrincipal user)
private sealed class TestApplicationFactory : WebApplicationFactory
{
- protected override IWebHostBuilder CreateWebHostBuilder()
+ protected override void ConfigureWebHost(IWebHostBuilder builder)
{
- return new WebHostBuilder()
- .UseSetting("TEST_CONTENTROOT_ASPNET_SECURITY_OPENID_PROVIDERS_TESTS", "."); // Use a dummy content root
+ base.ConfigureWebHost(builder);
+ builder.UseContentRoot("."); // Use a dummy content root
+ }
+
+ protected override IHostBuilder? CreateHostBuilder()
+ {
+ return new HostBuilder();
}
}
}