openssh: fix PKCS#11 tests instead of disabling softhsm

This is PRed upstream. While we usually do not apply patches to OpenSSH
itself prior to them being merged, this patch simply affects the tests
and lets us re-enable softhsm support.

See: https://github.com/NixOS/nixpkgs/issues/453782
See: https://github.com/openssh/openssh-portable/pull/602

pkgs/tools/networking/openssh/common.nix

@@ -65,6 +65,10 @@ stdenv.mkDerivation (finalAttrs: {
 
     # See discussion in https://github.com/NixOS/nixpkgs/pull/16966
     ./dont_create_privsep_path.patch
+
+    # See discussion in https://github.com/NixOS/nixpkgs/issues/453782 and
+    # https://github.com/openssh/openssh-portable/pull/602
+    ./fix_pkcs11_tests.patch
   ]
   ++ extraPatches;
 
@@ -148,8 +152,7 @@ stdenv.mkDerivation (finalAttrs: {
     openssl
   ]
   ++ lib.optional (!stdenv.hostPlatform.isDarwin) hostname
-  # TODO: softhsm is currently breaking the tests; see #453782
-  ++ lib.optional (false && !stdenv.hostPlatform.isDarwin && !stdenv.hostPlatform.isMusl) softhsm;
+  ++ lib.optional (!stdenv.hostPlatform.isDarwin && !stdenv.hostPlatform.isMusl) softhsm;
 
   preCheck = lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) (
     ''
@@ -198,15 +201,11 @@ stdenv.mkDerivation (finalAttrs: {
       # set up NIX_REDIRECTS for direct invocations
       set -a; source ~/.ssh/environment.base; set +a
     ''
-    # See softhsm in nativeCheckInputs above.
-    +
-      lib.optionalString
-        (!finalAttrs.doCheck && !stdenv.hostPlatform.isDarwin && !stdenv.hostPlatform.isMusl)
-        ''
-          # The extra tests check PKCS#11 interactions, which softhsm emulates with software only
-          substituteInPlace regress/test-exec.sh \
-            --replace-fail /usr/local/lib/softhsm/libsofthsm2.so ${lib.getLib softhsm}/lib/softhsm/libsofthsm2.so
-        ''
+    + lib.optionalString (!stdenv.hostPlatform.isDarwin && !stdenv.hostPlatform.isMusl) ''
+      # The extra tests check PKCS#11 interactions, which softhsm emulates with software only
+      substituteInPlace regress/test-exec.sh \
+        --replace-fail /usr/local/lib/softhsm/libsofthsm2.so ${lib.getLib softhsm}/lib/softhsm/libsofthsm2.so
+    ''
   );
   # integration tests hard to get working on darwin with its shaky
   # sandbox

pkgs/tools/networking/openssh/fix_pkcs11_tests.patch

@@ -0,0 +1,51 @@
+From 642218d8dd1ec79fa0c8db491fd46faa3ab026f7 Mon Sep 17 00:00:00 2001
+From: Morgan Jones <me@numin.it>
+Date: Tue, 21 Oct 2025 01:15:55 -0700
+Subject: [PATCH 1/2] test-exec: use -P for allowed PKCS#11 library when
+ starting agent
+
+If we just loaded a PKCS#11 library, we should allow it so the
+regression test can run.
+
+Fixes: https://github.com/NixOS/nixpkgs/issues/453782
+---
+ regress/test-exec.sh | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/regress/test-exec.sh b/regress/test-exec.sh
+index 5b0c91f3faa..30b3da8709d 100644
+--- a/regress/test-exec.sh
++++ b/regress/test-exec.sh
+@@ -1023,6 +1023,9 @@ p11_ssh_add() {
+ 
+ start_ssh_agent() {
+ 	EXTRA_AGENT_ARGS="$1"
++	if [ "$PKCS11_OK" = "yes" ]; then
++		EXTRA_AGENT_ARGS="${EXTRA_AGENT_ARGS} -P${TEST_SSH_PKCS11}"
++	fi
+ 	SSH_AUTH_SOCK="$OBJ/agent.sock"
+ 	export SSH_AUTH_SOCK
+ 	rm -f $SSH_AUTH_SOCK $OBJ/agent.log
+
+From 5ae735db7d81b38ee059d63a4011291cb4456aef Mon Sep 17 00:00:00 2001
+From: Morgan Jones <me@numin.it>
+Date: Tue, 21 Oct 2025 01:50:23 -0700
+Subject: [PATCH 2/2] test-exec: give more time for ssh-agent to start
+
+---
+ regress/test-exec.sh | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/regress/test-exec.sh b/regress/test-exec.sh
+index 30b3da8709d..56a7653b386 100644
+--- a/regress/test-exec.sh
++++ b/regress/test-exec.sh
+@@ -1034,7 +1034,7 @@ start_ssh_agent() {
+ 	    > $OBJ/agent.log 2>&1 &
+ 	AGENT_PID=$!
+ 	trap "kill $AGENT_PID" EXIT
+-	for x in 0 1 2 3 4 ; do
++	for x in $(seq 15); do
+ 		# Give it a chance to start
+ 		${SSHADD} -l > /dev/null 2>&1
+ 		r=$?