test: Reduce main sources of overhead in integration tests

The integration tests spend a lot of time on some ancilliary actions,
which slows them down: generating certificates, adding users, and
waiting for things to happen.

To improve the performance of those actions, this patch:

- Makes (most) tests use plain passwords (-20%)
- Adds a certificate cache to reuse certs (-10%)
- Tightens the sleep loops (-5%)

In aggregate, this patch results in a speedup of the integration tests
of ~30-40%.

Note that some of the tests required adjusting the username, because
`chasquid-util user-add` would convert them to lowercase as per PRECIS
mapping.

test/t-01-simple_local/run.sh

@@ -19,8 +19,8 @@ if chasquid -v=2 --logfile=.logs/chasquid.log --config_dir=config; then
 fi
 
 generate_certs_for testserver
-add_user user@testserver secretpassword
-add_user someone@testserver secretpassword
+chasquid-util-user-add user@testserver secretpassword
+chasquid-util-user-add someone@testserver secretpassword
 
 chasquid -v=2 --logfile=.logs/chasquid.log --config_dir=config &
 wait_until_ready 1025

test/t-06-idna/run.sh

@@ -22,11 +22,11 @@ minidns_bg --addr=":9053" -zones=zones >> .minidns.log 2>&1
 
 CONFDIR=A generate_certs_for srv-ñ
 CONFDIR=A add_user ñangapirí@srv-ñ antaño
-CONFDIR=A add_user nadaA@nadaA nadaA
+CONFDIR=A add_user nadaa@nadaA nadaA
 
 CONFDIR=B generate_certs_for srv-ü
 CONFDIR=B add_user pingüino@srv-ü velóz
-CONFDIR=B add_user nadaB@nadaB nadaB
+CONFDIR=B add_user nadab@nadaB nadaB
 
 mkdir -p .logs-A .logs-B
 

test/t-09-loop/run.sh

@@ -22,7 +22,7 @@ minidns_bg --addr=":9053" -zones=zones >> .minidns.log 2>&1
 #   userA -> aliasB -> aliasA -> aliasB -> ...
 
 CONFDIR=A generate_certs_for srv-A
-CONFDIR=A add_user userA@srv-A userA
+CONFDIR=A add_user usera@srv-A userA
 
 CONFDIR=B generate_certs_for srv-B
 

test/t-13-reload/run.sh

@@ -13,7 +13,7 @@ generate_certs_for testserver
 #
 
 # Start with the user with the wrong password, and no aliases.
-add_user someone@testserver password111
+chasquid-util-user-add someone@testserver password111
 rm -f config/domains/testserver/aliases
 
 mkdir -p .logs
@@ -28,7 +28,7 @@ fi
 
 # Change password, add an alias; then wait a bit more than the reload period
 # and try again.
-add_user someone@testserver password222
+chasquid-util-user-add someone@testserver password222
 echo "analias: someone" > config/domains/testserver/aliases
 sleep 0.2
 

test/t-14-tls_tracking/run.sh

@@ -20,10 +20,10 @@ minidns_bg --addr=":9053" -zones=zones >> .minidns.log 2>&1
 # B - listens on :2025, hosts srv-B
 
 CONFDIR=A generate_certs_for srv-A
-CONFDIR=A add_user userA@srv-A userA
+CONFDIR=A add_user usera@srv-A userA
 
 CONFDIR=B generate_certs_for srv-B
-CONFDIR=B add_user userB@srv-B userB
+CONFDIR=B add_user userb@srv-B userB
 
 rm -rf .data-A .data-B .mail .certs
 mkdir -p .logs-A .logs-B .mail .certs

test/t-16-spf/run.sh

@@ -18,10 +18,10 @@ export GOTAGS="dnsoverride"
 # B - listens on :2025, hosts srv-B
 
 CONFDIR=A generate_certs_for srv-A
-CONFDIR=A add_user userA@srv-A userA
+CONFDIR=A add_user usera@srv-A userA
 
 CONFDIR=B generate_certs_for srv-B
-CONFDIR=B add_user userB@srv-B userB
+CONFDIR=B add_user userb@srv-B userB
 
 rm -rf .data-A .data-B .mail .certs
 mkdir -p .logs-A .logs-B .mail .certs

test/util/lib.sh

@@ -60,7 +60,10 @@ function chasquid_cover() {
 			"$@"
 }
 
-function add_user() {
+# Add a user with chasquid-util. Because this is somewhat cryptographically
+# intensive, it can slow down the tests significantly, so most of the time we
+# use the simpler add_user (below) for testing purposes.
+function chasquid-util-user-add() {
 	CONFDIR="${CONFDIR:-config}"
 	DOMAIN=$(echo $1 | cut -d @ -f 2)
 	mkdir -p "${CONFDIR}/domains/$DOMAIN/"
@@ -71,6 +74,18 @@ function add_user() {
 		>> .add_user_logs
 }
 
+function add_user() {
+	CONFDIR="${CONFDIR:-config}"
+	USERNAME=$(echo $1 | cut -d @ -f 1)
+	DOMAIN=$(echo $1 | cut -d @ -f 2)
+	USERDB="${CONFDIR}/domains/$DOMAIN/users"
+	mkdir -p "${CONFDIR}/domains/$DOMAIN/"
+	if ! [ -f "${USERDB}" ] || ! grep -E -q "key:.*${USERNAME}" "${USERDB}"; then
+		echo "users:{ key: '${USERNAME}' value:{ plain:{ password: '$2' }}}" \
+			>> "${USERDB}"
+	fi
+}
+
 function dovecot-auth-cli() {
 	go run ${TBASE}/../../cmd/dovecot-auth-cli/dovecot-auth-cli.go "$@"
 }
@@ -160,14 +175,14 @@ function wait_until_ready() {
 	PORT=$1
 
 	while ! bash -c "true < /dev/tcp/localhost/$PORT" 2>/dev/null ; do
-		sleep 0.1
+		sleep 0.01
 	done
 }
 
 # Wait for the given file to exist.
 function wait_for_file() {
 	while ! [ -e ${1} ]; do
-		sleep 0.1
+		sleep 0.01
 	done
 }
 
@@ -176,18 +191,29 @@ function wait_until() {
 		if eval "$@"; then
 			return 0
 		fi
-		sleep 0.05
+		sleep 0.01
 	done
 }
 
 # Generate certs for the given hostname.
 function generate_certs_for() {
 	CONFDIR="${CONFDIR:-config}"
-	mkdir -p ${CONFDIR}/certs/${1}/
-	(
-		cd ${CONFDIR}/certs/${1}
-		generate_cert -ca -validfor=1h -host=${1}
-	)
+
+	# Generating certs is takes time and slows the tests down, so we keep
+	# a little cache that is common to all tests.
+	CACHEDIR="${TBASE}/../.generate_certs_cache"
+	mkdir -p "${CACHEDIR}"
+	touch -d "10 minutes ago" "${CACHEDIR}/.reference"
+	if [ "${CACHEDIR}/${1}/" -ot "${CACHEDIR}/.reference" ]; then
+		# Cache miss (either was not there, or was too old).
+		mkdir -p "${CACHEDIR}/${1}/"
+		(
+			cd "${CACHEDIR}/${1}/"
+			generate_cert -ca -validfor=1h -host=${1}
+		)
+	fi
+	mkdir -p "${CONFDIR}/certs/${1}/"
+	cp -p "${CACHEDIR}/${1}"/* "${CONFDIR}/certs/${1}/"
 }
 
 # Check the Python version, and skip if it's too old.