Implement basic IDNA support

This patch implements the first steps of support for IDNA (Internationalized
Domain Names for Applications).

Internally, we maintain the byte-agnostic representation, including
configuration.

We support receiving IDNA mail over SMTP, which we convert to unicode for
internal handling.

Local deliveries are still kept agnostic.

For sending over SMTP, we use IDNA for DNS resolution, but there are some
corner cases pending in the SMTP courier that are tied with SMTPUTF8 and will
be fixed in subsequent patches.

chasquid.go

@@ -609,6 +609,11 @@ func (c *Conn) MAIL(params string) (code int, msg string) {
 		if !strings.Contains(e.Address, "@") {
 			return 501, "sender address must contain a domain"
 		}
+
+		e.Address, err = envelope.IDNAToUnicode(e.Address)
+		if err != nil {
+			return 501, "malformed address (IDNA conversion failed)"
+		}
 	}
 
 	// Note some servers check (and fail) if we had a previous MAIL command,
@@ -641,6 +646,11 @@ func (c *Conn) RCPT(params string) (code int, msg string) {
 		return 501, "malformed address"
 	}
 
+	e.Address, err = envelope.IDNAToUnicode(e.Address)
+	if err != nil {
+		return 501, "malformed address (IDNA conversion failed)"
+	}
+
 	if c.mailFrom == "" {
 		return 503, "sender not yet given"
 	}

internal/courier/smtp.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/golang/glog"
+	"golang.org/x/net/idna"
 
 	"blitiri.com.ar/go/chasquid/internal/envelope"
 	"blitiri.com.ar/go/chasquid/internal/trace"
@@ -137,6 +138,11 @@ func lookupMX(domain string) (string, error) {
 		return v, nil
 	}
 
+	domain, err := idna.ToASCII(domain)
+	if err != nil {
+		return "", err
+	}
+
 	mxs, err := net.LookupMX(domain)
 	if err == nil {
 		if len(mxs) == 0 {

internal/envelope/envelope.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"strings"
 
+	"golang.org/x/net/idna"
+
 	"blitiri.com.ar/go/chasquid/internal/set"
 )
 
@@ -48,3 +50,27 @@ func AddHeader(data []byte, k, v string) []byte {
 	header := []byte(fmt.Sprintf("%s: %s\n", k, v))
 	return append(header, data...)
 }
+
+// Take an address with a potentially unicode domain, and convert it to ASCII
+// as per IDNA.
+// The user part is unchanged.
+func IDNAToASCII(addr string) (string, error) {
+	if addr == "<>" {
+		return addr, nil
+	}
+	user, domain := Split(addr)
+	domain, err := idna.ToASCII(domain)
+	return user + "@" + domain, err
+}
+
+// Take an address with an ASCII domain, and convert it to Unicode as per
+// IDNA.
+// The user part is unchanged.
+func IDNAToUnicode(addr string) (string, error) {
+	if addr == "<>" {
+		return addr, nil
+	}
+	user, domain := Split(addr)
+	domain, err := idna.ToUnicode(domain)
+	return user + "@" + domain, err
+}

test/t-06-idna/.gitignore

@@ -0,0 +1,2 @@
+# Ignore the configuration domain directories.
+?/domains

test/t-06-idna/A/chasquid.conf

@@ -0,0 +1,8 @@
+smtp_address: ":1025"
+submission_address: ":1587"
+monitoring_address: ":1099"
+
+mail_delivery_agent_bin: "test-mda"
+mail_delivery_agent_args: "%to%"
+
+data_dir: "../.data-A"

test/t-06-idna/B/chasquid.conf

@@ -0,0 +1,8 @@
+smtp_address: ":2025"
+submission_address: ":2587"
+monitoring_address: ":2099"
+
+mail_delivery_agent_bin: "test-mda"
+mail_delivery_agent_args: "%to%"
+
+data_dir: "../.data-B"

test/t-06-idna/from_A_to_B

@@ -0,0 +1,5 @@
+From: ñangapirí@srv-ñ
+To: pingüino@srv-ü
+Subject: Hola amigo pingüino!
+
+Que tal va la vida?

test/t-06-idna/from_B_to_A

@@ -0,0 +1,5 @@
+From: pingüino@srv-ü
+To: ñangapirí@srv-ñ
+Subject: Feliz primavera!
+
+Espero que florezcas feliz!

test/t-06-idna/hosts

@@ -0,0 +1,2 @@
+xn--srv--3ra localhost
+xn--srv--jqa localhost

test/t-06-idna/run.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -e
+. $(dirname ${0})/../util/lib.sh
+
+init
+
+rm -rf .data-A .data-B .mail
+
+skip_if_python_is_too_old
+
+# Two servers:
+# A - listens on :1025, hosts srv-ñ
+# B - listens on :2015, hosts srv-ü
+
+CONFDIR=A generate_certs_for srv-ñ
+CONFDIR=A add_user srv-ñ ñangapirí antaño
+CONFDIR=A add_user nadaA nadaA nadaA
+
+CONFDIR=B generate_certs_for srv-ü
+CONFDIR=B add_user srv-ü pingüino velóz
+CONFDIR=B add_user nadaB nadaB nadaB
+
+mkdir -p .logs-A .logs-B
+
+chasquid -v=2 --log_dir=.logs-A --config_dir=A \
+	--testing__outgoing_smtp_port=2025 &
+chasquid -v=2 --log_dir=.logs-B --config_dir=B \
+	--testing__outgoing_smtp_port=1025 &
+
+wait_until_ready 1025
+wait_until_ready 2025
+
+# Send from A to B.
+smtpc.py --server=localhost:1025 --user=nadaA@nadaA --password=nadaA \
+	< from_A_to_B
+
+wait_for_file .mail/pingüino@srv-ü
+mail_diff from_A_to_B .mail/pingüino@srv-ü
+
+# Send from B to A.
+smtpc.py --server=localhost:2025 --user=nadaB@nadaB --password=nadaB \
+	< from_B_to_A
+
+wait_for_file .mail/ñangapirí@srv-ñ
+mail_diff from_B_to_A .mail/ñangapirí@srv-ñ
+
+success

test/util/lib.sh

@@ -39,8 +39,11 @@ function chasquid() {
 }
 
 function add_user() {
+	CONFDIR="${CONFDIR:-config}"
+	mkdir -p "${CONFDIR}/domains/${1}/"
 	go run ${TBASE}/../../cmd/chasquid-util/chasquid-util.go \
-		adduser "config/domains/${1}/users" "${2}" --password "${3}" \
+		adduser "${CONFDIR}/domains/${1}/users" "${2}" \
+		--password "${3}" \
 		>> .add_user_logs
 }
 
@@ -52,6 +55,10 @@ function run_msmtp() {
 		msmtp -C msmtprc "$@"
 }
 
+function smtpc.py() {
+	${UTILDIR}/smtpc.py "$@"
+}
+
 function mail_diff() {
 	${UTILDIR}/mail_diff "$@"
 }
@@ -82,9 +89,21 @@ function wait_for_file() {
 
 # Generate certs for the given hostname.
 function generate_certs_for() {
-	mkdir -p config/certs/${1}/
+	CONFDIR="${CONFDIR:-config}"
+	mkdir -p ${CONFDIR}/certs/${1}/
 	(
-		cd config/certs/${1}
+		cd ${CONFDIR}/certs/${1}
 		generate_cert -ca -duration=1h -host=${1}
 	)
 }
+
+# Check the Python version, and skip if it's too old.
+# This will check against the version required for smtpc.py.
+function skip_if_python_is_too_old() {
+	# We need Python >= 3.5 to be able to use SMTPUTF8.
+	check='import sys; sys.exit(0 if sys.version_info >= (3, 5) else 1)'
+	if ! python3 -c "${check}"; then
+		skip "python3 >= 3.5 not available"
+		exit 0
+	fi
+}

test/util/smtpc.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+#
+# Simple SMTP client for testing purposes.
+
+import argparse
+import email.parser
+import email.policy
+import smtplib
+import sys
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--server", help="SMTP server to connect to")
+ap.add_argument("--user", help="Username to use in SMTP AUTH")
+ap.add_argument("--password", help="Password to use in SMTP AUTH")
+args = ap.parse_args()
+
+# Parse the email using the "default" policy, which is not really the default.
+# If unspecified, compat32 is used, which does not support UTF8.
+msg = email.parser.Parser(policy=email.policy.default).parse(sys.stdin)
+
+s = smtplib.SMTP(args.server)
+s.starttls()
+s.login(args.user, args.password)
+
+# Note this does NOT support non-ascii message payloads transparently (headers
+# are ok).
+s.send_message(msg)
+s.quit()
+
+