test: Unify (most) SMTP client calls

To send mails, today some tests use msmtp and others our internal smtpc.py.

This works, but msmtp slows down the tests significantly, and smtpc.py
is also not particularly fast, and also has some limitations.

This patch introduces a new SMTP client tool written in Go, and makes
almost all the tests use it.

Some tests still remain on msmtp, mainly for client-check compatibility.
It's likely that this will be moved in later patches to a separate
special-purpose test.

With this patch, integration tests take ~20% less time than before.

test/util/lib.sh

@@ -120,10 +120,6 @@ function run_msmtp() {
 		"${UTILDIR}/.msmtp-bin" -C msmtprc "$@"
 }
 
-function smtpc.py() {
-	"${UTILDIR}/smtpc.py" "$@"
-}
-
 function mail_diff() {
 	"${UTILDIR}/mail_diff" "$@"
 }
@@ -158,6 +154,11 @@ function fexp() {
 	"${UTILDIR}/fexp/fexp" "$@"
 }
 
+function smtpc() {
+	go-build-cached "${UTILDIR}/smtpc/"
+	"${UTILDIR}/smtpc/smtpc" "$@"
+}
+
 function timeout() {
 	MYPID=$$
 	(
@@ -232,16 +233,6 @@ function generate_certs_for() {
 	cp -p "${CACHEDIR}/$1"/* "${CONFDIR}/certs/$1/"
 }
 
-# Check the Python version, and skip if it's too old.
-# This will check against the version required for smtpc.py.
-function skip_if_python_is_too_old() {
-	# We need Python >= 3.5 to be able to use SMTPUTF8.
-	check='import sys; sys.exit(0 if sys.version_info >= (3, 5) else 1)'
-	if ! python3 -c "${check}" > /dev/null 2>&1; then
-		skip "python3 >= 3.5 not available"
-	fi
-}
-
 function chasquid_ram_peak() {
 	# Find the pid of the daemon, which we expect is running on the
 	# background somewhere within our current session.

test/util/smtpc.py

@@ -1,43 +0,0 @@
-#!/usr/bin/env python3
-#
-# Simple SMTP client for testing purposes.
-
-import argparse
-import email.parser
-import email.policy
-import re
-import smtplib
-import sys
-
-ap = argparse.ArgumentParser()
-ap.add_argument("--server", help="SMTP server to connect to")
-ap.add_argument("--user", help="Username to use in SMTP AUTH")
-ap.add_argument("--password", help="Password to use in SMTP AUTH")
-args = ap.parse_args()
-
-# Parse the email using the "default" policy, which is not really the default.
-# If unspecified, compat32 is used, which does not support UTF8.
-rawmsg = sys.stdin.buffer.read()
-msg = email.parser.Parser(policy=email.policy.default).parsestr(
-        rawmsg.decode('utf8'))
-
-s = smtplib.SMTP(args.server)
-s.starttls()
-if args.user:
-    s.login(args.user, args.password)
-
-# Send the raw message, not parsed, because the parser does not handle some
-# corner cases that well (for example, DKIM-Signature headers get mime-encoded
-# incorrectly).
-# Replace \n with \r\n, which is normally done by the library, but will not do
-# it in this case because we are giving it bytes and not a string (which we
-# cannot do because it tries to incorrectly escape the headers).
-crlfmsg = re.sub(br'(?:\r\n|\n|\r(?!\n))', b"\r\n", rawmsg)
-
-s.sendmail(
-        from_addr=msg['from'], to_addrs=msg.get_all('to'),
-        msg=crlfmsg,
-        mail_options=['SMTPUTF8'])
-s.quit()
-
-

test/util/smtpc/smtpc.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"flag"
+	"io"
+	"net"
+	"net/smtp"
+	"os"
+	"strings"
+)
+
+var (
+	addr = flag.String("addr", "", "Address of the SMTP server")
+
+	user     = flag.String("user", "", "Username to use in SMTP AUTH")
+	password = flag.String("password", "", "Password to use in SMTP AUTH")
+
+	from = flag.String("from", "", "From address to use in the message")
+
+	serverCert = flag.String("server_cert", "",
+		"Path to the server certificate to expect")
+
+	confPath = flag.String("c", "smtpc.conf",
+		"Path to the configuration file")
+)
+
+func main() {
+	flag.Parse()
+	loadConfig()
+
+	// Read message from stdin.
+	rawMsg, err := io.ReadAll(os.Stdin)
+	notnil(err)
+
+	// RCPT TO from the command line.
+	tos := make([]string, len(flag.Args()))
+	for i, to := range flag.Args() {
+		tos[i] = to
+	}
+
+	// Connect to the server.
+	var conn net.Conn
+	if *serverCert != "" {
+		cert := loadCert(*serverCert)
+		rootCAs := x509.NewCertPool()
+
+		rootCAs.AddCert(cert)
+		tlsConfig := &tls.Config{
+			ServerName: cert.DNSNames[0],
+			RootCAs:    rootCAs,
+		}
+
+		conn, err = tls.Dial("tcp", *addr, tlsConfig)
+		defer conn.Close()
+	} else {
+		conn, err = net.Dial("tcp", *addr)
+	}
+	notnil(err)
+
+	// Send the message.
+	client, err := smtp.NewClient(conn, *addr)
+	notnil(err)
+
+	if *user != "" {
+		auth := smtp.PlainAuth("", *user, *password, *addr)
+		err = client.Auth(auth)
+		notnil(err)
+	}
+
+	if *from == "" {
+		*from = *user
+	}
+	err = client.Mail(*from)
+	notnil(err)
+
+	for _, to := range tos {
+		err = client.Rcpt(to)
+		notnil(err)
+	}
+
+	w, err := client.Data()
+	notnil(err)
+	_, err = io.Copy(w, bytes.NewReader(rawMsg))
+	notnil(err)
+	err = w.Close()
+	notnil(err)
+
+	err = client.Quit()
+	notnil(err)
+}
+
+func loadConfig() {
+	data, err := os.ReadFile(*confPath)
+	if errors.Is(err, os.ErrNotExist) {
+		return
+	}
+	notnil(err)
+
+	for _, line := range strings.Split(string(data), "\n") {
+		k, v, ok := strings.Cut(line, " ")
+		if !ok {
+			continue
+		}
+
+		k = strings.TrimSpace(k)
+
+		// Set the flag but only if it wasn't already set.
+		// Command-line flags take precedence.
+		isSet := false
+		flag.Visit(func(f *flag.Flag) {
+			if f.Name == k {
+				isSet = true
+			}
+		})
+		if !isSet {
+			flag.Lookup(k).Value.Set(strings.TrimSpace(v))
+		}
+	}
+}
+
+func loadCert(path string) *x509.Certificate {
+	data, err := os.ReadFile(path)
+	notnil(err)
+
+	block, _ := pem.Decode(data)
+
+	cert, err := x509.ParseCertificate(block.Bytes)
+	notnil(err)
+
+	return cert
+}
+
+func notnil(err error) {
+	if err != nil {
+		panic(err)
+	}
+}