[Foundation] Implement NSUrlSessionHandler proxy support. Fixes #14632

tests/dotnet/UnitTests/expected/MacCatalyst-NativeAOT-TrimmableStatic-size.txt

@@ -1,9 +1,9 @@
-AppBundleSize: 13,916,973 bytes (13,590.8 KB = 13.3 MB)
+AppBundleSize: 13,933,496 bytes (13,606.9 KB = 13.3 MB)
 # The following list of files and their sizes is just informational / for review, and isn't used in the test:
 Contents/Info.plist:
-    1,109 bytes (1.1 KB = 0.0 MB)
+    1,040 bytes (1.0 KB = 0.0 MB)
 Contents/MacOS/SizeTestApp:
-    13,913,960 bytes (13,587.9 KB = 13.3 MB)
+    13,930,552 bytes (13,604.1 KB = 13.3 MB)
 Contents/MonoBundle/runtimeconfig.bin:
     1,896 bytes (1.9 KB = 0.0 MB)
 Contents/PkgInfo:

tests/dotnet/UnitTests/expected/MacOSX-NativeAOT-TrimmableStatic-size.txt

@@ -1,9 +1,9 @@
-AppBundleSize: 32,590,412 bytes (31,826.6 KB = 31.1 MB)
+AppBundleSize: 32,623,366 bytes (31,858.8 KB = 31.1 MB)
 # The following list of files and their sizes is just informational / for review, and isn't used in the test:
 Contents/Info.plist:
-    740 bytes (0.7 KB = 0.0 MB)
+    718 bytes (0.7 KB = 0.0 MB)
 Contents/MacOS/SizeTestApp:
-    29,501,112 bytes (28,809.7 KB = 28.1 MB)
+    29,534,088 bytes (28,841.9 KB = 28.2 MB)
 Contents/MonoBundle/libSystem.Globalization.Native.dylib:
     267,872 bytes (261.6 KB = 0.3 MB)
 Contents/MonoBundle/libSystem.IO.Compression.Native.dylib:

tests/dotnet/UnitTests/expected/TVOS-NativeAOT-TrimmableStatic-size.txt

@@ -1,10 +1,10 @@
-AppBundleSize: 12,531,409 bytes (12,237.7 KB = 12.0 MB)
+AppBundleSize: 12,564,243 bytes (12,269.8 KB = 12.0 MB)
 # The following list of files and their sizes is just informational / for review, and isn't used in the test:
 Info.plist:
-    1,128 bytes (1.1 KB = 0.0 MB)
+    1,106 bytes (1.1 KB = 0.0 MB)
 PkgInfo:
     8 bytes (0.0 KB = 0.0 MB)
 runtimeconfig.bin:
     1,889 bytes (1.8 KB = 0.0 MB)
 SizeTestApp:
-    12,528,384 bytes (12,234.8 KB = 11.9 MB)
+    12,561,240 bytes (12,266.8 KB = 12.0 MB)

tests/dotnet/UnitTests/expected/iOS-NativeAOT-TrimmableStatic-size.txt

@@ -1,10 +1,10 @@
-AppBundleSize: 14,136,248 bytes (13,804.9 KB = 13.5 MB)
+AppBundleSize: 14,152,698 bytes (13,821.0 KB = 13.5 MB)
 # The following list of files and their sizes is just informational / for review, and isn't used in the test:
 Info.plist:
-    1,152 bytes (1.1 KB = 0.0 MB)
+    1,130 bytes (1.1 KB = 0.0 MB)
 PkgInfo:
     8 bytes (0.0 KB = 0.0 MB)
 runtimeconfig.bin:
     1,888 bytes (1.8 KB = 0.0 MB)
 SizeTestApp:
-    14,133,200 bytes (13,802.0 KB = 13.5 MB)
+    14,149,672 bytes (13,818.0 KB = 13.5 MB)

tests/monotouch-test/System.Net.Http/MessageHandlers.cs

@@ -838,7 +838,7 @@ public void TestNSUrlSessionHandlerOptionalClientCertificate ()
 		{
 			NWListener? listener = null;
 			try {
-				listener = CreateNWTlsListener (requireClientCert: false);
+				listener = TlsTestServer.CreateNWTlsListener (requireClientCert: false);
 				var port = listener.Port;
 
 				var done = TestRuntime.TryRunAsync (TimeSpan.FromSeconds (30), async () => {
@@ -861,7 +861,7 @@ public void TestNSUrlSessionHandlerDetectMissingClientCertificate ()
 		{
 			NWListener? listener = null;
 			try {
-				listener = CreateNWTlsListener (requireClientCert: true);
+				listener = TlsTestServer.CreateNWTlsListener (requireClientCert: true);
 				var port = listener.Port;
 
 				var done = TestRuntime.TryRunAsync (TimeSpan.FromSeconds (30), async () => {
@@ -889,7 +889,7 @@ public void TestNSUrlSessionHandlerDetectMissingClientCertificateOptOut ()
 			NWListener? listener = null;
 			try {
 				AppContext.SetSwitch ("Foundation.NSUrlSessionHandler.NoMissingCertificateHandling", true);
-				listener = CreateNWTlsListener (requireClientCert: true);
+				listener = TlsTestServer.CreateNWTlsListener (requireClientCert: true);
 				var port = listener.Port;
 
 				var done = TestRuntime.TryRunAsync (TimeSpan.FromSeconds (30), async () => {
@@ -912,77 +912,6 @@ public void TestNSUrlSessionHandlerDetectMissingClientCertificateOptOut ()
 			}
 		}
 
-		static NWListener CreateNWTlsListener (bool requireClientCert)
-		{
-			var (pfxData, pfxPassword) = CreateSelfSignedServerCertificatePfx ();
-			using var secIdentity = SecIdentity.Import (pfxData, pfxPassword);
-			using var secIdentity2 = new SecIdentity2 (secIdentity);
-			using var readyEvent = new ManualResetEventSlim (false);
-			NWError? listenerError = null;
-
-			var parameters = NWParameters.CreateSecureTcp (
-				configureTls: tlsOptions => {
-					var tls = (NWProtocolTlsOptions) tlsOptions;
-					var secOptions = tls.ProtocolOptions;
-					secOptions.SetLocalIdentity (secIdentity2);
-					secOptions.SetPeerAuthenticationRequired (requireClientCert);
-				});
-			using var localEndpoint = NWEndpoint.Create ("127.0.0.1", "0");
-			parameters.LocalEndpoint = localEndpoint;
-
-			var listener = NWListener.Create (parameters);
-			parameters.Dispose ();
-
-			listener.SetQueue (CoreFoundation.DispatchQueue.DefaultGlobalQueue);
-
-			listener.SetStateChangedHandler ((state, error) => {
-				if (state == NWListenerState.Failed)
-					listenerError = error;
-				if (state == NWListenerState.Ready || state == NWListenerState.Failed)
-					readyEvent.Set ();
-			});
-
-			listener.SetNewConnectionHandler (connection => {
-				connection.SetQueue (CoreFoundation.DispatchQueue.DefaultGlobalQueue);
-				connection.SetStateChangeHandler ((connState, connError) => {
-					if (connState == NWConnectionState.Ready) {
-						// Read the HTTP request (just consume it), then send a response
-						connection.ReceiveReadOnlyData (1, 4096, (data, context, isComplete, error) => {
-							var response = Encoding.UTF8.GetBytes ("HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nOK");
-							connection.Send (response, NWContentContext.FinalMessage, true, sendError => {
-								connection.Cancel ();
-							});
-						});
-					}
-				});
-				connection.Start ();
-			});
-
-			listener.Start ();
-
-			if (!readyEvent.Wait (TimeSpan.FromSeconds (10)))
-				throw new TimeoutException ("NWListener did not become ready in time.");
-
-			if (listenerError is not null)
-				throw new InvalidOperationException ($"NWListener failed to start: {listenerError}");
-
-			return listener;
-		}
-
-		static (byte [] Data, string Password) CreateSelfSignedServerCertificatePfx ()
-		{
-			using var rsa = RSA.Create (2048);
-			var certRequest = new CertificateRequest (
-				"CN=localhost", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
-			var sanBuilder = new SubjectAlternativeNameBuilder ();
-			sanBuilder.AddIpAddress (IPAddress.Loopback);
-			sanBuilder.AddDnsName ("localhost");
-			certRequest.CertificateExtensions.Add (sanBuilder.Build ());
-			var cert = certRequest.CreateSelfSigned (DateTimeOffset.UtcNow.AddDays (-1), DateTimeOffset.UtcNow.AddYears (1));
-			var password = Guid.NewGuid ().ToString ();
-			return (cert.Export (X509ContentType.Pfx, password), password);
-		}
-
 		sealed class RedirectBasicAuthServer : IDisposable {
 			readonly bool crossOrigin;
 			readonly HttpListener originListener;

tests/monotouch-test/System.Net.Http/NSUrlSessionHandlerTest.cs

@@ -18,6 +18,12 @@ namespace MonoTests.System.Net.Http {
 	[Preserve (AllMembers = true)]
 	public class NSUrlSessionHandlerTest {
 
+		// The proxy tests below use in-process servers bound to 127.0.0.1, so local network
+		// connections should be reliable and we don't want to hide any failures by ignoring them
+		// in CI. Set this to true to restore the usual "ignore transient network failures in CI"
+		// behavior if these tests ever turn out to be flaky on the bots.
+		bool ignoreLocalOnlyCIFailures = false;
+
 		// https://github.com/dotnet/macios/issues/23958
 		[Test]
 		public void DecompressedResponseDoesNotHaveContentEncodingOrContentLength ()
@@ -399,6 +405,148 @@ public void StreamReadAsyncCallerCancellationThrowsOperationCanceledException ()
 			}
 		}
 
+		[Test]
+		public void ProxyRoutesRequestsThroughProxy ()
+		{
+			using var proxy = new ProxyTestServer ();
+
+			HttpStatusCode? statusCode = null;
+			bool viaProxy = false;
+
+			var done = TestRuntime.TryRunAsync (TimeSpan.FromSeconds (30), async () => {
+				using var handler = new NSUrlSessionHandler ();
+				handler.Proxy = new WebProxy (proxy.Url);
+				Assert.That (handler.UseProxy, Is.True, "UseProxy default");
+				Assert.That (handler.SupportsProxy, Is.True, "SupportsProxy");
+				using var client = new HttpClient (handler);
+				var response = await client.GetAsync (NetworkResources.Httpbin.GetUrl).ConfigureAwait (false);
+				statusCode = response.StatusCode;
+				viaProxy = response.Headers.Contains ("Via-Test-Proxy");
+			}, out var ex);
+
+			if (!done) {
+				if (ignoreLocalOnlyCIFailures)
+					TestRuntime.IgnoreInCI ("Transient localhost server failure - ignore in CI");
+				Assert.Inconclusive ("Request timed out.");
+			}
+			if (ignoreLocalOnlyCIFailures)
+				TestRuntime.IgnoreInCIIfBadNetwork (ex);
+			Assert.That (ex, Is.Null, $"Exception: {ex}");
+			Assert.That (statusCode, Is.EqualTo (HttpStatusCode.OK), "Status code");
+			Assert.That (viaProxy, Is.True, "Response should have gone through the test proxy");
+			Assert.That (proxy.AuthenticatedRequestCount, Is.GreaterThan (0), "Proxy should have forwarded at least one request");
+		}
+
+		[Test]
+		public void ProxyWithCredentialsAuthenticatesWithProxy ()
+		{
+			const string proxyUser = "proxyuser";
+			const string proxyPass = "proxypass";
+
+			// NSUrlSession only delivers a proxy authentication challenge to the delegate for CONNECT
+			// tunnels (i.e. HTTPS destinations); for a plain HTTP forward proxy it returns the 407
+			// directly to the caller. So we route an HTTPS request through the proxy's CONNECT support.
+			// We also request a non-local hostname: CFNetwork bypasses the proxy for localhost HTTPS
+			// destinations, so we must use a hostname that isn't local. The proxy tunnels every CONNECT
+			// to our local TLS test server regardless of the requested host.
+			Network.NWListener? destination = null;
+			HttpStatusCode? statusCode = null;
+
+			try {
+				destination = TlsTestServer.CreateNWTlsListener (requireClientCert: false);
+				var destinationPort = destination.Port;
+				using var proxy = new ProxyTestServer (proxyUser, proxyPass, forceTunnelPort: (int) destinationPort);
+
+				var done = TestRuntime.TryRunAsync (TimeSpan.FromSeconds (30), async () => {
+					using var handler = new NSUrlSessionHandler ();
+					handler.Proxy = new WebProxy (proxy.Url) {
+						Credentials = new NetworkCredential (proxyUser, proxyPass),
+					};
+					handler.TrustOverrideForUrl = (sender, url, trust) => true;
+					using var client = new HttpClient (handler);
+					var response = await client.GetAsync ("https://proxy-tunnel-target.example/").ConfigureAwait (false);
+					statusCode = response.StatusCode;
+				}, out var ex);
+
+				if (!done) {
+					if (ignoreLocalOnlyCIFailures)
+						TestRuntime.IgnoreInCI ("Transient localhost server failure - ignore in CI");
+					Assert.Inconclusive ("Request timed out.");
+				}
+				if (ignoreLocalOnlyCIFailures)
+					TestRuntime.IgnoreInCIIfBadNetwork (ex);
+				Assert.That (ex, Is.Null, $"Exception: {ex}");
+				Assert.That (statusCode, Is.EqualTo (HttpStatusCode.OK), $"Status code (proxy credentials should have been used); status={statusCode}, requestCount={proxy.RequestCount}, authRequestCount={proxy.AuthenticatedRequestCount}");
+				Assert.That (proxy.AuthenticatedRequestCount, Is.GreaterThan (0), "Proxy should have established an authenticated tunnel");
+			} finally {
+				destination?.Cancel ();
+				destination?.Dispose ();
+			}
+		}
+
+		[Test]
+		public void ProxyWithDefaultProxyCredentialsAuthenticatesWithProxy ()
+		{
+			const string proxyUser = "proxyuser";
+			const string proxyPass = "proxypass";
+
+			// See ProxyWithCredentialsAuthenticatesWithProxy for why we use an HTTPS (CONNECT) request
+			// with a non-local hostname and tunnel every CONNECT to a local TLS test server.
+			Network.NWListener? destination = null;
+			HttpStatusCode? statusCode = null;
+
+			try {
+				destination = TlsTestServer.CreateNWTlsListener (requireClientCert: false);
+				var destinationPort = destination.Port;
+				using var proxy = new ProxyTestServer (proxyUser, proxyPass, forceTunnelPort: (int) destinationPort);
+
+				var done = TestRuntime.TryRunAsync (TimeSpan.FromSeconds (30), async () => {
+					using var handler = new NSUrlSessionHandler ();
+					handler.Proxy = new WebProxy (proxy.Url);
+					handler.DefaultProxyCredentials = new NetworkCredential (proxyUser, proxyPass);
+					handler.TrustOverrideForUrl = (sender, url, trust) => true;
+					using var client = new HttpClient (handler);
+					var response = await client.GetAsync ("https://proxy-tunnel-target.example/").ConfigureAwait (false);
+					statusCode = response.StatusCode;
+				}, out var ex);
+
+				if (!done) {
+					if (ignoreLocalOnlyCIFailures)
+						TestRuntime.IgnoreInCI ("Transient localhost server failure - ignore in CI");
+					Assert.Inconclusive ("Request timed out.");
+				}
+				if (ignoreLocalOnlyCIFailures)
+					TestRuntime.IgnoreInCIIfBadNetwork (ex);
+				Assert.That (ex, Is.Null, $"Exception: {ex}");
+				Assert.That (statusCode, Is.EqualTo (HttpStatusCode.OK), $"Status code (default proxy credentials should have been used); status={statusCode}, requestCount={proxy.RequestCount}, authRequestCount={proxy.AuthenticatedRequestCount}");
+				Assert.That (proxy.AuthenticatedRequestCount, Is.GreaterThan (0), "Proxy should have established an authenticated tunnel");
+			} finally {
+				destination?.Cancel ();
+				destination?.Dispose ();
+			}
+		}
+
+		[Test]
+		public void ProxyPropertiesBehaveCorrectly ()
+		{
+			using var handler = new NSUrlSessionHandler ();
+			Assert.That (handler.Proxy, Is.Null, "Proxy default");
+			Assert.That (handler.UseProxy, Is.True, "UseProxy default");
+			Assert.That (handler.SupportsProxy, Is.True, "SupportsProxy");
+			Assert.That (handler.DefaultProxyCredentials, Is.Null, "DefaultProxyCredentials default");
+
+			var proxy = new WebProxy ("http://127.0.0.1:8888");
+			handler.Proxy = proxy;
+			Assert.That (handler.Proxy, Is.SameAs (proxy), "Proxy set");
+
+			handler.UseProxy = false;
+			Assert.That (handler.UseProxy, Is.False, "UseProxy set to false");
+
+			var credentials = new NetworkCredential ("user", "pass");
+			handler.DefaultProxyCredentials = credentials;
+			Assert.That (handler.DefaultProxyCredentials, Is.SameAs (credentials), "DefaultProxyCredentials set");
+		}
+
 		static HttpListener? StartListenerOnAvailablePort (out int listeningPort)
 		{
 			// IANA suggested range for dynamic or private ports