wolfsshd: tighten the run registry and ports

The registry now holds only daemons that are still running, and it is
cleaned up however the run ends. Both private-daemon ports come from the
runner, so they stay inside the block it probed even when --port moves
the shared daemon off it.

- drop a pid from the registry once stop_wolfsshd has stopped it, so the
  end-of-run sweep cannot reach a pid since recycled by another run
- run the sweep and the registry cleanup from an EXIT trap: every early
  exit used to skip them and leave the file in /tmp
- export WOLFSSHD_PRIVDROP_PORT rather than re-deriving the offset in
  sshd_privdrop_fail_test.sh, and correct that script's usage message
- quote the arguments to create_sshd_config.sh: an empty USER shifted
  the port into $1, silently leaving the daemon on 22222
- name the three tests that still read the whole process table, which
  are what keeps two concurrent runs from being fully independent

apps/wolfsshd/test/run_all_sshd_tests.sh

@@ -78,6 +78,13 @@ done
 # The range deliberately starts above 22226: other CI steps in this repo bind
 # 22222, 22225 and 22226 while the suite is not running, and a daemon this
 # suite leaks must not be able to sit on one of them.
+#
+# Not yet per run: sshd_term_close_test.sh counts "pgrep wolfsshd" before and
+# after its connection, sshd_sftp_idle_cpu_test.sh picks the first wolfsshd
+# that is new since its connection, and sshd_stdin_stall_test.sh sums CPU
+# ticks over every wolfsshd on the machine. All three read the whole process
+# table, so a second run's connection children can perturb them. Two runs at
+# once otherwise pass; these are the remaining single-run-per-host tests.
 PORT_BLOCK_FIRST=22300
 PORT_BLOCK_SIZE=8
 PORT_BLOCK_COUNT=64
@@ -125,8 +132,11 @@ LOCAL_PORT="${TEST_PORT:-$PORT_BASE}"
 STRICTMODES_PORT=$((PORT_BASE + 1))
 UPN_PORT=$((PORT_BASE + 2))
 HOSTKEY_PERM_PORT=$((PORT_BASE + 3))
-# Read by sshd_ossh_cert_test.sh, which starts a daemon of its own.
+# Read by the two tests that start a daemon of their own. They are exported
+# rather than derived from the port passed to the test, so they stay inside the
+# probed block even when --port moves the shared daemon off it.
 export WOLFSSHD_TEST_PORT=$((PORT_BASE + 4))
+export WOLFSSHD_PRIVDROP_PORT=$((PORT_BASE + 5))
 
 # Registry of the daemons started during this run, appended to by
 # start_wolfsshd in every test script that sources start_sshd.sh. The exit
@@ -136,6 +146,34 @@ WOLFSSHD_TEST_PIDFILE=`mktemp 2>/dev/null` \
     || WOLFSSHD_TEST_PIDFILE=`mktemp -t sshdpids`
 export WOLFSSHD_TEST_PIDFILE
 
+# Teardown safety net: the start/stop pairs below stop each daemon they start,
+# but background test daemons survive across CI steps that share this runner,
+# and a later step (the valgrind "memory after close down" check) binds a port
+# of its own. Make sure no daemon this run started lingers when the script
+# exits, and that the registry does not outlive it either.
+#
+# It is a trap, not a block at the bottom of the file, because the script exits
+# early on a bad --match, a setup failure, a daemon that will not start and
+# every test failure -- none of which would reach the bottom.
+#
+# Scoped to the pids in the registry, not to the wolfsshd name: "pkill -x
+# wolfsshd" here matched every other run's daemon too, so on a shared runner
+# whichever job finished first took down the other's. A port-matched pkill is
+# not an option for the shared daemon -- its port comes from its config file,
+# so it never appears on the command line to match against.
+#
+# USING_LOCAL_HOST is unset on the early exits that precede "source
+# ./start_sshd.sh", so stop_all_wolfsshd is never called before it is defined.
+# Every step ends in "|| true": a failing command in an EXIT trap becomes the
+# script's exit status, which would turn a passing run red.
+run_teardown() {
+    if [ "$USING_LOCAL_HOST" == 1 ]; then
+        stop_all_wolfsshd || true
+    fi
+    rm -f "$WOLFSSHD_TEST_PIDFILE" || true
+}
+trap run_teardown EXIT
+
 TOTAL=0
 SKIPPED=0
 # Set as the last statement of each branch that runs tests, and checked before
@@ -168,7 +206,7 @@ fi
 # setup
 set -e
 ./create_authorized_test_file.sh
-./create_sshd_config.sh $USER $LOCAL_PORT
+./create_sshd_config.sh "$USER" "$LOCAL_PORT"
 set +e
 
 if [ ! -z "$TEST_HOST" ] && [ ! -z "$TEST_PORT" ]; then
@@ -618,22 +656,6 @@ else
     RUN_COMPLETE=1
 fi
 
-# Teardown safety net: the start/stop pairs above stop each daemon they start,
-# but background test daemons survive across CI steps that share this runner,
-# and a later step (the valgrind "memory after close down" check) binds a port
-# of its own. Make sure no daemon this run started lingers when the script
-# exits. Harmless when nothing is running.
-#
-# Scoped to the pids in the registry, not to the wolfsshd name: "pkill -x
-# wolfsshd" here matched every other run's daemon too, so on a shared runner
-# whichever job finished first took down the other's. A port-matched pkill is
-# not an option for the shared daemon -- its port comes from its config file,
-# so it never appears on the command line to match against.
-if [ "$USING_LOCAL_HOST" == 1 ]; then
-    stop_all_wolfsshd
-fi
-rm -f "$WOLFSSHD_TEST_PIDFILE"
-
 if [ "$RUN_COMPLETE" != 1 ]; then
     printf "ERROR: test run aborted before all tests ran\n"
     exit 1

apps/wolfsshd/test/sshd_privdrop_fail_test.sh

@@ -7,8 +7,9 @@
 # Drives all three dropping subsystems: exec/shell, sftp, scp.
 
 if [ -z "$1" ] || [ -z "$2" ]; then
-    echo "expecting host and port as arguments"
-    echo "./sshd_privdrop_fail_test.sh 127.0.0.1 22222"
+    echo "expecting host and the runner's shared port as arguments;"
+    echo "this test binds WOLFSSHD_PRIVDROP_PORT, or that port + 5"
+    echo "./sshd_privdrop_fail_test.sh 127.0.0.1 22300"
     exit 1
 fi
 
@@ -19,10 +20,10 @@ USER=`whoami`
 TEST_HOST="$1"
 
 # Own daemon on a dedicated port for isolation from the runner's shared daemon.
-# Offset from the port passed in rather than a constant, so the isolation holds
-# against another run of the suite on the same host and not just against this
-# run's shared daemon. The runner reserves the offset as part of its block.
-TEST_PORT=$(( $2 + 5 ))
+# The runner reserves this port as part of the block it probed and exports it,
+# so the isolation holds against another run of the suite on the same host.
+# Standalone, fall back to an offset from the port passed in.
+TEST_PORT="${WOLFSSHD_PRIVDROP_PORT:-$(( $2 + 5 ))}"
 
 SSHD_BIN="../wolfsshd"
 if [ ! -x "$SSHD_BIN" ]; then

apps/wolfsshd/test/start_sshd.sh

@@ -185,6 +185,18 @@ stop_wolfsshd() {
             sleep 0.1
         done
 
+        # Drop it from the run registry now that it is stopped. Left there, it
+        # would still be a candidate for the end-of-run sweep, which can only
+        # ask whether some wolfsshd holds that pid today -- and a concurrent
+        # run forks one per connection, so a recycled pid would be that run's
+        # daemon. A run accumulates about nine of these, all dead but one.
+        if [ -n "$WOLFSSHD_TEST_PIDFILE" ] && [ -f "$WOLFSSHD_TEST_PIDFILE" ]; then
+            grep -vx -- "$PID" "$WOLFSSHD_TEST_PIDFILE" \
+                > "$WOLFSSHD_TEST_PIDFILE.new" 2>/dev/null || true
+            mv -f "$WOLFSSHD_TEST_PIDFILE.new" "$WOLFSSHD_TEST_PIDFILE" \
+                2>/dev/null || rm -f "$WOLFSSHD_TEST_PIDFILE.new"
+        fi
+
         # Cleared so a second call -- an EXIT trap after an explicit stop -- is
         # a no-op rather than a kill of whatever pid has since been recycled.
         PID=""