[net10.0] [msbuild] Only decompress binding resource packages remotely. Fixes #23346. (#23859)

This hopefully fixes a problem where the build would run into MAX_PATH issues
on Windows, because now we won't be extracting any xcframework files on
Windows at all.

Also:

* Change the default to always create compressed binding resource packages.
  This will ensure NuGets will successfully extract on Windows, because
  they'll contain the compressed binding resource package, instead of the
  individual files.
* Speed up the ResolveNativeReferences task:
   * Only run it if there's anything to do.
   * Only pass paths that contain references to resolve - this will speed up
     the build on Windows, because it massively reduces the amount of files
     that have to be transferred to the remote mac.

Fixes #23346.

Author: rolfbjarne

docs/building-apps/build-properties.md

@@ -279,8 +279,8 @@ The `CompressBindingResourcePackage` property specifies whether to create a zip
 
 The possible values are:
 
-* `auto`: create a zip file if a native reference contains symlinks (which is typical on macOS and Mac Catalyst, but rare on iOS and tvOS).
-* `true`: create a zipe file
+* `auto`: automatically decide the best option (currently a zip file is always created, but once Visual Studio supports long paths on Windows, this may change to only zip binding resource packages with symlinks).
+* `true`: create a zip file
 * `false`: create a directory
 
 The default is `auto`.

dotnet/targets/Xamarin.Shared.Sdk.targets

@@ -918,10 +918,44 @@
 		</ItemGroup>
 	</Target>
 
+	<!-- When copying files into the app bundle, the final paths can be *long*. This is not a problem on macOS, but Windows still lives in a world where MAX_PATH is unreasonable,
+	     so we have to come up with some hopefully not too unreasonable workarounds.
+
+	     We want the _CopyDirectoriesToBundle task to have correct Inputs and Outputs (to support incremental builds properly), but to avoid ending up
+	     with Outputs paths that cause problems on Windows, we use a stamp file as the Output, and the stamp file is a hashed version of the actual Output location
+
+	     This target will compute that stamp file location.
+	-->
+	<Target Name="_CreateStampLocationForCopyDirectoriesToBundle">
+		<ComputeHashForItems
+			Input="@(_DirectoriesToPublish)"
+			InputMetadata="Identity;TargetDirectory"
+			OutputMetadata="StampHash"
+		>
+			<Output TaskParameter="Output" ItemName="_DirectoriesToPublishWithStampHash" />
+		</ComputeHashForItems>
+		<ItemGroup>
+			<_DirectoriesToPublish />
+			<_DirectoriesToPublish Include="@(_DirectoriesToPublishWithStampHash)">
+				<StampLocation>$(DeviceSpecificIntermediateOutputPath)copieddirectories\%(_DirectoriesToPublishWithStampHash.StampHash)</StampLocation>
+			</_DirectoriesToPublish>
+
+			<_StampFilesToDelete Include="@(_DirectoriesToPublish -> '%(StampLocation)')" Condition="!Exists('%(_DirectoriesToPublish.TargetDirectory)')">
+				<SourceDirectory>%(_DirectoriesToPublish.SourceDirectory)</SourceDirectory>
+				<TargetDirectory>%(_DirectoriesToPublish.TargetDirectory)</TargetDirectory>
+			</_StampFilesToDelete>
+		</ItemGroup>
+
+		<Delete
+			SessionId="$(BuildSessionId)"
+			Files="@(_StampFilesToDelete)"
+		/>
+	</Target>
+
 	<Target Name="_CopyDirectoriesToBundle"
-		DependsOnTargets="_CollectDecompressedPlugins;_ComputeFrameworkFilesToPublish;_CollectDecompressedXpcServices"
+		DependsOnTargets="_CollectDecompressedPlugins;_ComputeFrameworkFilesToPublish;_CollectDecompressedXpcServices;_CreateStampLocationForCopyDirectoriesToBundle"
 		Inputs="@(_DirectoriesToPublish)"
-		Outputs="@(_DirectoriesToPublish -> '%(TargetDirectory)/%(Filename)')"
+		Outputs="@(_DirectoriesToPublish -> '%(StampLocation)')"
 		>
 
 		<!-- Ask ditto to not copy architectures we don't care about. This can be overridden by passing 'SkipLibraryThinning=true' -->
@@ -942,6 +976,8 @@
 			Source="%(_DirectoriesToPublish.SourceDirectory)"
 			Destination="%(_DirectoriesToPublish.TargetDirectory)"
 			TouchDestinationFiles="true"
+			StampFile="%(_DirectoriesToPublish.StampLocation)"
+			CreateOutputFiles="false"
 		/>
 	</Target>
 
@@ -2168,14 +2204,16 @@
 		</ItemGroup>
 
 		<!-- resolve any .xcframeworks and binding resource packages -->
-		<!-- Skip doing this when not building on macOS if we're not a remoteable platform (i.e. iOS), because we want such builds to succeed (even if they produce non-working output), and ResolveNativeReferences will fail if running into symlinks (which happens for macOS and Mac Catalyst builds) -->
+		<ItemGroup>
+			<_NativeReferencesToResolve Include="@(_UnresolvedXCFrameworks);@(_AppleBindingResourcePackage);@(_CompressedAppleFrameworks);@(_CompressedAppleBindingResourcePackage)" />
+		</ItemGroup>
 		<ResolveNativeReferences
 			SessionId="$(BuildSessionId)"
-			Condition="'$(IsMacEnabled)' == 'true' And '$(_IsNonMacOSBuildForNonRemotablePlatform)' != 'true'"
+			Condition="'$(IsMacEnabled)' == 'true' And @(_NativeReferencesToResolve->Count()) &gt; 0"
 			Architectures="$(TargetArchitectures)"
 			FrameworksDirectory="$(_AppFrameworksRelativePath)"
 			IntermediateOutputPath="$(DeviceSpecificIntermediateOutputPath)"
-			NativeReferences="@(_UnresolvedXCFrameworks);@(_AppleBindingResourcePackage);@(_CompressedAppleFrameworks);@(_CompressedAppleBindingResourcePackage)"
+			NativeReferences="@(_NativeReferencesToResolve)"
 			SdkIsSimulator="$(_SdkIsSimulator)"
 			TargetFrameworkMoniker="$(_ComputedTargetFrameworkMoniker)"
 			>

msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx

@@ -1613,4 +1613,12 @@
             {1}: the name of the property that is empty (SourceFiles, Items, FrameworkToPublish, etc.).
         </comment>
     </data>
+
+    <data name="W7160" xml:space="preserve">
+        <value>Creating a compressed binding resource package to avoid MAX_PATH problems on Windows.</value>
+    </data>
+
+    <data name="W7161" xml:space="preserve">
+        <value>Not creating a compressed binding resource package, because all the native references are already compressed.</value>
+    </data>
 </root>

msbuild/Xamarin.MacDev.Tasks/Decompress.cs

@@ -29,6 +29,19 @@ public static bool IsCompressed (string path)
 			return path.EndsWith (".zip", StringComparison.OrdinalIgnoreCase);
 		}
 
+		static string CanonicalizeZipEntryPath (string path)
+		{
+			// The directory separator character is supposed to be '/' on all platforms in zip files,
+			// but that's not always the case, so canonicalize the path.
+			return path.Replace ('\\', '/');
+		}
+
+		static ZipArchiveEntry? GetCanonicalizedEntry (ZipArchive archive, string path)
+		{
+			var canonicalizedPath = CanonicalizeZipEntryPath (path);
+			return archive.Entries.SingleOrDefault (e => string.Equals (CanonicalizeZipEntryPath (e.FullName), canonicalizedPath, StringComparison.Ordinal));
+		}
+
 		/// <summary>
 		/// Finds a file from either a directory or a zip file.
 		/// </summary>
@@ -45,7 +58,7 @@ public static bool IsCompressed (string path)
 					return null;
 				}
 				using var zip = ZipFile.OpenRead (resources);
-				var contentEntry = zip.GetEntry (relativeFilePath.Replace ('\\', '/')); // directory separator character is '/' on all platforms in zip files.
+				var contentEntry = GetCanonicalizedEntry (zip, relativeFilePath);
 				if (contentEntry is null) {
 					log.LogWarning (MSBStrings.W7106 /* Expected a file named '{1}' in the zip file {0}. */, resources, relativeFilePath);
 					return null;
@@ -144,18 +157,20 @@ static bool TryDecompressUsingUnzip (TaskLoggingHelper log, string zip, string r
 			if (!string.IsNullOrEmpty (resource)) {
 				using var archive = ZipFile.OpenRead (zip);
 				resource = resource.Replace ('\\', zipDirectorySeparator);
-				var entry = archive.GetEntry (resource);
+				var entry = GetCanonicalizedEntry (archive, resource);
 				if (entry is null) {
-					entry = archive.GetEntry (resource + zipDirectorySeparator);
+					entry = GetCanonicalizedEntry (archive, resource + zipDirectorySeparator);
 					if (entry is null) {
 						log.LogError (MSBStrings.E7112 /* Could not find the file or directory '{0}' in the zip file '{1}'. */, resource, zip);
 						return false;
 					}
 				}
 
 				var zipPattern = entry.FullName;
-				if (zipPattern.Length > 0 && zipPattern [zipPattern.Length - 1] == zipDirectorySeparator) {
-					zipPattern += "*";
+				if (zipPattern.Length > 0) {
+					var last = zipPattern [zipPattern.Length - 1];
+					if (last == '\\' || last == '/')
+						zipPattern += "*";
 				}
 
 				args.Add (zipPattern);
@@ -178,7 +193,7 @@ static bool TryDecompressUsingSystemIOCompression (TaskLoggingHelper log, string
 			using var archive = ZipFile.OpenRead (zip);
 			foreach (var entry in archive.Entries) {
 				cancellationToken?.ThrowIfCancellationRequested ();
-				var entryPath = entry.FullName;
+				var entryPath = CanonicalizeZipEntryPath (entry.FullName);
 				if (entryPath.Length == 0)
 					continue;
 
@@ -331,16 +346,17 @@ static bool TryCompressUsingSystemIOCompression (TaskLoggingHelper log, string z
 
 			var rootDirLength = workingDirectory.Length + 1;
 			foreach (var resource in resourcePaths) {
-				log.LogMessage (MessageImportance.Low, $"Procesing {resource}");
+				log.LogMessage (MessageImportance.Low, $"Processing {resource}");
 				if (Directory.Exists (resource)) {
+					if (rootDirLength < resource.Length) {
+						var resourceZipName = resource.Substring (rootDirLength);
+						archive.CreateEntry (CanonicalizeZipEntryPath (resourceZipName) + zipDirectorySeparator);
+					}
 					var entries = Directory.GetFileSystemEntries (resource, "*", SearchOption.AllDirectories);
 					var entriesWithZipName = entries.Select (v => new { Path = v, ZipName = v.Substring (rootDirLength) });
 					foreach (var entry in entriesWithZipName) {
 						if (Directory.Exists (entry.Path)) {
-							if (entries.Where (v => v.StartsWith (entry.Path, StringComparison.Ordinal)).Count () == 1) {
-								// this is a directory with no files inside, we need to create an entry with a trailing directory separator.
-								archive.CreateEntry (entry.ZipName + zipDirectorySeparator);
-							}
+							archive.CreateEntry (CanonicalizeZipEntryPath (entry.ZipName) + zipDirectorySeparator);
 						} else {
 							WriteFileToZip (log, archive, entry.Path, entry.ZipName, maxCompression);
 						}
@@ -359,7 +375,7 @@ static bool TryCompressUsingSystemIOCompression (TaskLoggingHelper log, string z
 
 		static void WriteFileToZip (TaskLoggingHelper log, ZipArchive archive, string path, string zipName, bool maxCompression)
 		{
-			var zipEntry = archive.CreateEntry (zipName, maxCompression ? SmallestCompressionLevel : CompressionLevel.Optimal);
+			var zipEntry = archive.CreateEntry (CanonicalizeZipEntryPath (zipName), maxCompression ? SmallestCompressionLevel : CompressionLevel.Optimal);
 			using var fs = File.OpenRead (path);
 			using var zipStream = zipEntry.Open ();
 			fs.CopyTo (zipStream);

msbuild/Xamarin.MacDev.Tasks/Tasks/Codesign.cs

@@ -194,10 +194,42 @@ bool NeedsCodesign (ITaskItem? [] sortedItems, int index, string stampFileConten
 				return true;
 			}
 
+			var signatureFile = GetSignatureFileLocation (item);
+			if (!string.IsNullOrEmpty (signatureFile) && !File.Exists (signatureFile)) {
+				Log.LogMessage (MessageImportance.Low, "The signature file '{0}' for the item '{0}' does not exist, so the item must be codesigned.", signatureFile, item.ItemSpec);
+				return true;
+			}
+
 			Log.LogMessage (MessageImportance.Low, "The stamp file '{0}' for the item '{1}' is up-to-date, so the item does not need to be codesigned.", stampFile, item.ItemSpec);
 			return false;
 		}
 
+		string? GetSignatureFileLocation (ITaskItem? item)
+		{
+			var path = item?.ItemSpec;
+#if NET
+			if (string.IsNullOrEmpty (path))
+#else
+			if (path is null || string.IsNullOrEmpty (path))
+#endif
+				return null;
+
+			if (path.EndsWith (".framework", StringComparison.OrdinalIgnoreCase) || path.EndsWith (".xpc", StringComparison.OrdinalIgnoreCase)) {
+				switch (Platform) {
+				case ApplePlatform.iOS:
+				case ApplePlatform.TVOS:
+					return Path.Combine (path, "_CodeSignature", "CodeResources");
+				case ApplePlatform.MacOSX:
+				case ApplePlatform.MacCatalyst:
+					return Path.Combine (path, "Versions", "A", "_CodeSignature", "CodeResources");
+				default:
+					throw new InvalidOperationException (string.Format (MSBStrings.InvalidPlatform, Platform));
+				}
+			}
+
+			return null;
+		}
+
 		bool ParseBoolean (ITaskItem item, string metadataName, bool fallbackValue)
 		{
 			var metadataValue = item.GetMetadata (metadataName);

msbuild/Xamarin.MacDev.Tasks/Tasks/ComputeHashForItems.cs

@@ -0,0 +1,65 @@
+#nullable enable
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Text;
+using System.Threading;
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+using Xamarin.Localization.MSBuild;
+using Xamarin.Messaging.Build.Client;
+using Xamarin.Utils;
+
+namespace Xamarin.MacDev.Tasks {
+	// This task will iterate over each input item, compute a hash value for all the specified metadata in the input items, and then set the specified output metadata to the hashed value
+	public class ComputeHashForItems : XamarinTask {
+		[Required]
+		public ITaskItem [] Input { get; set; } = Array.Empty<ITaskItem> ();
+
+		// The metadata in each input item to use as input for the hash algorithm.
+		[Required]
+		public ITaskItem [] InputMetadata { get; set; } = Array.Empty<ITaskItem> ();
+
+		// The name of the metadata where to store the computed hashed value
+		[Required]
+		public string OutputMetadata { get; set; } = string.Empty;
+
+		// The output items. This will be Input, where each item will also have 'OutputMetadata' set to the computed hash value.
+		[Output]
+		public ITaskItem [] Output { get; set; } = Array.Empty<ITaskItem> ();
+
+		public override bool Execute ()
+		{
+			if (Input.Length == 0)
+				return true;
+
+			using var sha = CreateHashAlgorithm ();
+
+			var buffer = new List<byte> ();
+			for (var i = 0; i < Input.Length; i++) {
+				var input = Input [i];
+				buffer.Clear ();
+				foreach (var im in InputMetadata) {
+					buffer.AddRange (Encoding.UTF8.GetBytes (input.GetMetadata (im.ItemSpec)));
+				}
+				var hashBytes = sha.ComputeHash (buffer.ToArray ());
+				var hash = string.Join ("", hashBytes.Select (b => $"{b:x2}"));
+				input.SetMetadata (OutputMetadata, hash);
+			}
+
+			Output = Input;
+
+			return !Log.HasLoggedErrors;
+		}
+
+		HashAlgorithm CreateHashAlgorithm ()
+		{
+			return SHA256.Create ();
+		}
+	}
+}

msbuild/Xamarin.MacDev.Tasks/Tasks/CreateBindingResourcePackage.cs

@@ -65,9 +65,16 @@ public override bool Execute ()
 			} else if (string.Equals (Compress, "false", StringComparison.OrdinalIgnoreCase)) {
 				compress = false;
 			} else if (string.Equals (Compress, "auto", StringComparison.OrdinalIgnoreCase)) {
-				compress = ContainsSymlinks (NativeReferences);
-				if (compress)
+				if (ContainsSymlinks (NativeReferences)) {
 					Log.LogMessage (MessageImportance.Low, MSBStrings.W7085 /* "Creating a compressed binding resource package because there are symlinks in the input." */);
+					compress = true;
+				} else if (NativeReferences.All (v => v.ItemSpec.EndsWith (".zip", StringComparison.OrdinalIgnoreCase))) {
+					compress = false;
+					Log.LogMessage (MessageImportance.Low, MSBStrings.W7161 /* "Not creating a compressed binding resource package, because all the native references are already compressed." */);
+				} else {
+					Log.LogMessage (MessageImportance.Low, MSBStrings.W7160 /* "Creating a compressed binding resource package to avoid MAX_PATH problems on Windows." */);
+					compress = true;
+				}
 			} else {
 				Log.LogError (MSBStrings.E7086 /* "The value '{0}' is invalid for the Compress property. Valid values: 'true', 'false' or 'auto'." */, Compress);
 			}