aliases: Implement "via" aliases

This patch implements "via" aliases, which let us explicitly select a
server to use for delivery.

This feature is useful in different scenarios, such as a secondary MX
server that forwards all incoming email to a primary.

For now, it is experimental and the syntax and semantics are subject to
change.

test/t-22-forward_via/.gitignore

@@ -0,0 +1,3 @@
+primary/**/users
+secondary/**/users
+external/**/users

test/t-22-forward_via/content

@@ -0,0 +1,7 @@
+Subject: Los espejos
+
+Yo que sentí el horror de los espejos
+no sólo ante el cristal impenetrable
+donde acaba y empieza, inhabitable,
+un imposible espacio de reflejos
+

test/t-22-forward_via/expected-chain-1

@@ -0,0 +1,54 @@
+Authentication-Results: primary
+	;spf=pass (matched mx)
+	;dkim=pass  header.b=????????????  header.d=dodo
+Received-SPF: pass (matched mx)
+Received: from *
+	by primary (chasquid) with ESMTPS
+	tls TLS_*
+	(over SMTP, TLS-1.3, envelope from "chain-1-4+fwd_from=chain-1-3+fwd_from=user222=dodo=kiwi@dodo")
+	; *
+Authentication-Results: secondary
+	;spf=pass (matched mx)
+	;dkim=pass  header.b=????????????  header.d=dodo
+Received-SPF: pass (matched mx)
+Received: from *
+	by secondary (chasquid) with ESMTPS
+	tls TLS_*
+	(over SMTP, TLS-1.3, envelope from "chain-1-3+fwd_from=user222=dodo@kiwi")
+	; *
+Authentication-Results: external
+	;spf=pass (matched mx)
+	;dkim=pass  header.b=????????????  header.d=dodo
+Received-SPF: pass (matched mx)
+Received: from *
+	by external (chasquid) with ESMTPS
+	tls TLS_*
+	(over SMTP, TLS-1.3, envelope from "user222@dodo")
+	; *
+Authentication-Results: primary
+	;spf=pass (matched mx)
+	;dkim=pass  header.b=????????????  header.d=dodo
+Received-SPF: pass (matched mx)
+Received: from *
+	by primary (chasquid) with ESMTPS
+	tls TLS_*
+	(over SMTP, TLS-1.3, envelope from "user222@dodo")
+	; *
+Received: from localhost
+	by secondary (chasquid) with ESMTPSA
+	tls TLS_*
+	(over submission+TLS, TLS-1.3, envelope from "user222@dodo")
+	; *
+DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed;
+	d=dodo; s=sel-secondary-1; t=*
+	h=subject:from:subject:date:to:cc:message-id;
+	bh=*
+	b=*
+	  *
+Subject: Los espejos
+
+Yo que sentí el horror de los espejos
+no sólo ante el cristal impenetrable
+donde acaba y empieza, inhabitable,
+un imposible espacio de reflejos
+

test/t-22-forward_via/expected-external-user333@kiwi

@@ -0,0 +1,27 @@
+Authentication-Results: external
+	;spf=pass (matched mx)
+	;dkim=pass  header.b=????????????  header.d=dodo
+Received-SPF: pass (matched mx)
+Received: from *
+	by external (chasquid) with ESMTPS
+	tls TLS_*
+	(over SMTP, TLS-1.3, envelope from "user222@dodo")
+	; *
+Received: from localhost
+	by secondary (chasquid) with ESMTPSA
+	tls TLS_*
+	(over submission+TLS, TLS-1.3, envelope from "user222@dodo")
+	; *
+DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed;
+	d=dodo; s=sel-secondary-1; t=*
+	h=subject:from:subject:date:to:cc:message-id;
+	bh=*
+	b=*
+	  *
+Subject: Los espejos
+
+Yo que sentí el horror de los espejos
+no sólo ante el cristal impenetrable
+donde acaba y empieza, inhabitable,
+un imposible espacio de reflejos
+

test/t-22-forward_via/expected-primary-user111@dodo

@@ -0,0 +1,27 @@
+Authentication-Results: primary
+	;spf=pass (matched mx)
+	;dkim=pass  header.b=????????????  header.d=dodo
+Received-SPF: pass (matched mx)
+Received: from *
+	by primary (chasquid) with ESMTPS
+	tls TLS_*
+	(over SMTP, TLS-1.3, envelope from "user222@dodo")
+	; *
+Received: from localhost
+	by secondary (chasquid) with ESMTPSA
+	tls TLS_*
+	(over submission+TLS, TLS-1.3, envelope from "user222@dodo")
+	; *
+DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed;
+	d=dodo; s=sel-secondary-1; t=*
+	h=subject:from:subject:date:to:cc:message-id;
+	bh=*
+	b=*
+	  *
+Subject: Los espejos
+
+Yo que sentí el horror de los espejos
+no sólo ante el cristal impenetrable
+donde acaba y empieza, inhabitable,
+un imposible espacio de reflejos
+

test/t-22-forward_via/external/chasquid.conf

@@ -0,0 +1,10 @@
+smtp_address: "127.0.0.20:1025"
+submission_address: ":3587"
+submission_over_tls_address: ":3465"
+monitoring_address: ":3099"
+
+mail_delivery_agent_bin: "test-mda"
+mail_delivery_agent_args: "external:%to%"
+
+data_dir: "../.data-external"
+mail_log_path: "../.logs/external-mail_log"

test/t-22-forward_via/external/domains/kiwi/aliases

@@ -0,0 +1,2 @@
+# Part 3 of chain-1 (see run.sh for the full structure).
+chain-1-3: chain-1-4@dodo via secondary

test/t-22-forward_via/primary/chasquid.conf

@@ -0,0 +1,10 @@
+smtp_address: "127.0.0.10:1025"
+submission_address: ":1587"
+submission_over_tls_address: ":1465"
+monitoring_address: ":1099"
+
+mail_delivery_agent_bin: "test-mda"
+mail_delivery_agent_args: "primary:%to%"
+
+data_dir: "../.data-primary"
+mail_log_path: "../.logs/primary-mail_log"

test/t-22-forward_via/primary/domains/dodo/aliases

@@ -0,0 +1,5 @@
+# Part 2 of chain-1 (see run.sh for the full structure).
+chain-1-2: chain-1-3@kiwi
+
+# Part 5 of chain-1 (see run.sh for the full structure).
+chain-1-5: user111@dodo

test/t-22-forward_via/run.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -e
+. "$(dirname "$0")/../util/lib.sh"
+
+init
+check_hostaliases
+
+rm -rf .data-primary .data-secondary .data-external .mail
+rm -f {primary,secondary,external}/domains/*/dkim:*.pem
+
+# Build with the DNS override, so we can fake DNS records.
+export GOTAGS="dnsoverride"
+
+# Two servers for the same domain "dodo":
+# primary - listens on 127.0.0.10:1025
+# secondary - listens on 127.0.0.11:1025
+#
+# One server for domain "kiwi":
+# external - listens on 127.0.0.20:1025
+
+CONFDIR=primary generate_certs_for primary
+CONFDIR=primary add_user user111@dodo user111
+CONFDIR=primary chasquid-util dkim-keygen --algo=ed25519 \
+	dodo sel-primary-1 > /dev/null
+
+CONFDIR=secondary generate_certs_for secondary
+CONFDIR=secondary add_user user222@dodo user222
+CONFDIR=secondary chasquid-util dkim-keygen --algo=ed25519 \
+	dodo sel-secondary-1 > /dev/null
+
+CONFDIR=external generate_certs_for external
+CONFDIR=external add_user user333@kiwi user333
+CONFDIR=external chasquid-util dkim-keygen --algo=ed25519 \
+	kiwi sel-external-1 > /dev/null
+
+
+# Launch minidns in the background using our configuration.
+# Augment the zones with the dkim ones.
+cp zones .zones
+CONFDIR=primary   chasquid-util dkim-dns dodo | sed 's/"//g' >> .zones
+CONFDIR=secondary chasquid-util dkim-dns dodo | sed 's/"//g' >> .zones
+CONFDIR=external  chasquid-util dkim-dns kiwi | sed 's/"//g' >> .zones
+minidns_bg --addr=":9053" -zones=.zones >> .minidns.log 2>&1
+
+
+mkdir -p .logs
+
+chasquid -v=2 --logfile=.logs/primary-chasquid.log --config_dir=primary \
+	--testing__dns_addr=127.0.0.1:9053 \
+	--testing__outgoing_smtp_port=1025 &
+chasquid -v=2 --logfile=.logs/secondary-chasquid.log --config_dir=secondary \
+	--testing__dns_addr=127.0.0.1:9053 \
+	--testing__outgoing_smtp_port=1025 &
+chasquid -v=2 --logfile=.logs/external-chasquid.log --config_dir=external \
+	--testing__dns_addr=127.0.0.1:9053 \
+	--testing__outgoing_smtp_port=1025 &
+
+wait_until "true < /dev/tcp/127.0.0.10/1025" 2>/dev/null
+wait_until "true < /dev/tcp/127.0.0.11/1025" 2>/dev/null
+wait_until "true < /dev/tcp/127.0.0.20/1025" 2>/dev/null
+wait_until_ready 9053
+
+# Connect to secondary, send an email to user111@dodo (which exists only in
+# the primary).  It should be forwarded to the primary.
+# Note this also verifies SRS is correct (by comparing the Received headers),
+# and that DKIM signatures are generated by secondary, and successfully
+# validated by primary.
+smtpc -c=smtpc-secondary.conf user111@dodo < content
+wait_for_file .mail/primary:user111@dodo
+mail_diff expected-primary-user111@dodo .mail/primary:user111@dodo
+
+# Connect to the secondary, send an email to user333@kiwi (which exists only
+# in external). It should be DKIM signed and delivered to the external server.
+# This is a normal delivery.
+smtpc -c=smtpc-secondary.conf user333@kiwi < content
+wait_for_file .mail/external:user333@kiwi
+mail_diff expected-external-user333@kiwi .mail/external:user333@kiwi
+
+# Connect to the secondary, send an email to chain-1@dodo, which has a long
+# alias chain:
+#   secondary: chain-1-1@dodo  ->  chain-1-2@dodo via primary
+#   primary:   chain-1-2@dodo  ->  chain-1-3@kiwi
+#   external:  chain-1-3@kiwi  ->  chain-1-4@dodo via secondary
+#   secondary: chain-1-4@dodo  ->  chain-1-5@dodo via primary
+#   primary:   chain-1-5@dodo  ->  user111@dodo
+rm .mail/primary:user111@dodo
+smtpc -c=smtpc-secondary.conf chain-1-1@dodo < content
+wait_for_file .mail/primary:user111@dodo
+mail_diff expected-chain-1 .mail/primary:user111@dodo
+
+success

test/t-22-forward_via/secondary/chasquid.conf

@@ -0,0 +1,10 @@
+smtp_address: "127.0.0.11:1025"
+submission_address: ":2587"
+submission_over_tls_address: ":2465"
+monitoring_address: ":2099"
+
+mail_delivery_agent_bin: "test-mda"
+mail_delivery_agent_args: "secondary:%to%"
+
+data_dir: "../.data-secondary"
+mail_log_path: "../.logs/secondary-mail_log"

test/t-22-forward_via/secondary/domains/dodo/aliases

@@ -0,0 +1,8 @@
+# Part 1 of chain-1 (see run.sh for the full structure).
+chain-1-1: chain-1-2@dodo via primary
+
+# Part 4 chain-1 (see run.sh for the full structure).
+chain-1-4: chain-1-5@dodo via primary
+
+# Forward all email to the primary server.
+*: * via primary

test/t-22-forward_via/smtpc-secondary.conf

@@ -0,0 +1,4 @@
+addr localhost:2465
+server_cert secondary/certs/secondary/fullchain.pem
+user user222@dodo
+password user222

test/t-22-forward_via/zones

@@ -0,0 +1,15 @@
+primary    A     127.0.0.10
+secondary  A     127.0.0.11
+external   A     127.0.0.20
+
+dodo       MX 10 primary
+dodo       MX 20 secondary
+
+# We need to use mx/8 because the source address will usually be 127.0.0.1,
+# not 127.0.0.10 or 127.0.0.11.
+# TODO: Once we support specifying a sender IP address, we should use that
+# and remove the /8.
+dodo       TXT v=spf1 mx/8 -all
+
+kiwi       MX 10 external
+kiwi       TXT v=spf1 mx/8 -all

test/util/mail_diff

@@ -57,11 +57,27 @@ def msg_equals(expected, msg):
         if expected[h] == "*":
             continue
 
-        if not flexible_eq(val, msg[h]):
-            print("Header %r differs:" % h)
-            print("Exp: %r" % val)
-            print("Got: %r" % msg[h])
-            diff = True
+        msg_hdr_vals = msg.get_all(h, [])
+        if len(msg_hdr_vals) == 1:
+            if not flexible_eq(val, msg[h]):
+                print("Header %r differs:" % h)
+                print("Exp: %r" % val)
+                print("Got: %r" % msg[h])
+                diff = True
+        else:
+            # We have multiple values for this header, so we need to check each
+            # one, and only return a diff if none of them match.
+            # Note this will result in a false positive if two headers match
+            # the same expected one, but this is good enough for now.
+            for msg_hdr_val in msg_hdr_vals:
+                if flexible_eq(val, msg_hdr_val):
+                    break
+            else:
+                print("Header %r differs, no matching header found" % h)
+                print("Exp: %r" % val)
+                for i, msg_hdr_val in enumerate(msg_hdr_vals):
+                    print("Got %d: %r" % (i, msg_hdr_val))
+                diff = True
 
     if diff:
         return False