From f767b83fe0202bce41b0ddd6d9f429f5ff864f39 Mon Sep 17 00:00:00 2001 From: Alberto Bertogli Date: Sat, 1 Oct 2016 20:20:41 +0100 Subject: [PATCH] Implement basic IDNA support This patch implements the first steps of support for IDNA (Internationalized Domain Names for Applications). Internally, we maintain the byte-agnostic representation, including configuration. We support receiving IDNA mail over SMTP, which we convert to unicode for internal handling. Local deliveries are still kept agnostic. For sending over SMTP, we use IDNA for DNS resolution, but there are some corner cases pending in the SMTP courier that are tied with SMTPUTF8 and will be fixed in subsequent patches. --- chasquid.go | 10 +++++++ internal/courier/smtp.go | 6 +++++ internal/envelope/envelope.go | 26 ++++++++++++++++++ test/t-06-idna/.gitignore | 2 ++ test/t-06-idna/A/chasquid.conf | 8 ++++++ test/t-06-idna/B/chasquid.conf | 8 ++++++ test/t-06-idna/from_A_to_B | 5 ++++ test/t-06-idna/from_B_to_A | 5 ++++ test/t-06-idna/hosts | 2 ++ test/t-06-idna/run.sh | 48 ++++++++++++++++++++++++++++++++++ test/util/lib.sh | 25 +++++++++++++++--- test/util/smtpc.py | 30 +++++++++++++++++++++ 12 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 test/t-06-idna/.gitignore create mode 100644 test/t-06-idna/A/chasquid.conf create mode 100644 test/t-06-idna/B/chasquid.conf create mode 100644 test/t-06-idna/from_A_to_B create mode 100644 test/t-06-idna/from_B_to_A create mode 100644 test/t-06-idna/hosts create mode 100755 test/t-06-idna/run.sh create mode 100755 test/util/smtpc.py diff --git a/chasquid.go b/chasquid.go index e88254a..9f6d447 100644 --- a/chasquid.go +++ b/chasquid.go @@ -609,6 +609,11 @@ func (c *Conn) MAIL(params string) (code int, msg string) { if !strings.Contains(e.Address, "@") { return 501, "sender address must contain a domain" } + + e.Address, err = envelope.IDNAToUnicode(e.Address) + if err != nil { + return 501, "malformed address (IDNA conversion failed)" + } } // Note some servers check (and fail) if we had a previous MAIL command, @@ -641,6 +646,11 @@ func (c *Conn) RCPT(params string) (code int, msg string) { return 501, "malformed address" } + e.Address, err = envelope.IDNAToUnicode(e.Address) + if err != nil { + return 501, "malformed address (IDNA conversion failed)" + } + if c.mailFrom == "" { return 503, "sender not yet given" } diff --git a/internal/courier/smtp.go b/internal/courier/smtp.go index 847d5ad..206bf9f 100644 --- a/internal/courier/smtp.go +++ b/internal/courier/smtp.go @@ -8,6 +8,7 @@ import ( "time" "github.com/golang/glog" + "golang.org/x/net/idna" "blitiri.com.ar/go/chasquid/internal/envelope" "blitiri.com.ar/go/chasquid/internal/trace" @@ -137,6 +138,11 @@ func lookupMX(domain string) (string, error) { return v, nil } + domain, err := idna.ToASCII(domain) + if err != nil { + return "", err + } + mxs, err := net.LookupMX(domain) if err == nil { if len(mxs) == 0 { diff --git a/internal/envelope/envelope.go b/internal/envelope/envelope.go index d0eb1d2..c05ec6e 100644 --- a/internal/envelope/envelope.go +++ b/internal/envelope/envelope.go @@ -6,6 +6,8 @@ import ( "fmt" "strings" + "golang.org/x/net/idna" + "blitiri.com.ar/go/chasquid/internal/set" ) @@ -48,3 +50,27 @@ func AddHeader(data []byte, k, v string) []byte { header := []byte(fmt.Sprintf("%s: %s\n", k, v)) return append(header, data...) } + +// Take an address with a potentially unicode domain, and convert it to ASCII +// as per IDNA. +// The user part is unchanged. +func IDNAToASCII(addr string) (string, error) { + if addr == "<>" { + return addr, nil + } + user, domain := Split(addr) + domain, err := idna.ToASCII(domain) + return user + "@" + domain, err +} + +// Take an address with an ASCII domain, and convert it to Unicode as per +// IDNA. +// The user part is unchanged. +func IDNAToUnicode(addr string) (string, error) { + if addr == "<>" { + return addr, nil + } + user, domain := Split(addr) + domain, err := idna.ToUnicode(domain) + return user + "@" + domain, err +} diff --git a/test/t-06-idna/.gitignore b/test/t-06-idna/.gitignore new file mode 100644 index 0000000..5e5dd84 --- /dev/null +++ b/test/t-06-idna/.gitignore @@ -0,0 +1,2 @@ +# Ignore the configuration domain directories. +?/domains diff --git a/test/t-06-idna/A/chasquid.conf b/test/t-06-idna/A/chasquid.conf new file mode 100644 index 0000000..2b10846 --- /dev/null +++ b/test/t-06-idna/A/chasquid.conf @@ -0,0 +1,8 @@ +smtp_address: ":1025" +submission_address: ":1587" +monitoring_address: ":1099" + +mail_delivery_agent_bin: "test-mda" +mail_delivery_agent_args: "%to%" + +data_dir: "../.data-A" diff --git a/test/t-06-idna/B/chasquid.conf b/test/t-06-idna/B/chasquid.conf new file mode 100644 index 0000000..0c73544 --- /dev/null +++ b/test/t-06-idna/B/chasquid.conf @@ -0,0 +1,8 @@ +smtp_address: ":2025" +submission_address: ":2587" +monitoring_address: ":2099" + +mail_delivery_agent_bin: "test-mda" +mail_delivery_agent_args: "%to%" + +data_dir: "../.data-B" diff --git a/test/t-06-idna/from_A_to_B b/test/t-06-idna/from_A_to_B new file mode 100644 index 0000000..a18f0d4 --- /dev/null +++ b/test/t-06-idna/from_A_to_B @@ -0,0 +1,5 @@ +From: ñangapirí@srv-ñ +To: pingüino@srv-ü +Subject: Hola amigo pingüino! + +Que tal va la vida? diff --git a/test/t-06-idna/from_B_to_A b/test/t-06-idna/from_B_to_A new file mode 100644 index 0000000..6eb98a6 --- /dev/null +++ b/test/t-06-idna/from_B_to_A @@ -0,0 +1,5 @@ +From: pingüino@srv-ü +To: ñangapirí@srv-ñ +Subject: Feliz primavera! + +Espero que florezcas feliz! diff --git a/test/t-06-idna/hosts b/test/t-06-idna/hosts new file mode 100644 index 0000000..05e33f4 --- /dev/null +++ b/test/t-06-idna/hosts @@ -0,0 +1,2 @@ +xn--srv--3ra localhost +xn--srv--jqa localhost diff --git a/test/t-06-idna/run.sh b/test/t-06-idna/run.sh new file mode 100755 index 0000000..2c04c2e --- /dev/null +++ b/test/t-06-idna/run.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +set -e +. $(dirname ${0})/../util/lib.sh + +init + +rm -rf .data-A .data-B .mail + +skip_if_python_is_too_old + +# Two servers: +# A - listens on :1025, hosts srv-ñ +# B - listens on :2015, hosts srv-ü + +CONFDIR=A generate_certs_for srv-ñ +CONFDIR=A add_user srv-ñ ñangapirí antaño +CONFDIR=A add_user nadaA nadaA nadaA + +CONFDIR=B generate_certs_for srv-ü +CONFDIR=B add_user srv-ü pingüino velóz +CONFDIR=B add_user nadaB nadaB nadaB + +mkdir -p .logs-A .logs-B + +chasquid -v=2 --log_dir=.logs-A --config_dir=A \ + --testing__outgoing_smtp_port=2025 & +chasquid -v=2 --log_dir=.logs-B --config_dir=B \ + --testing__outgoing_smtp_port=1025 & + +wait_until_ready 1025 +wait_until_ready 2025 + +# Send from A to B. +smtpc.py --server=localhost:1025 --user=nadaA@nadaA --password=nadaA \ + < from_A_to_B + +wait_for_file .mail/pingüino@srv-ü +mail_diff from_A_to_B .mail/pingüino@srv-ü + +# Send from B to A. +smtpc.py --server=localhost:2025 --user=nadaB@nadaB --password=nadaB \ + < from_B_to_A + +wait_for_file .mail/ñangapirí@srv-ñ +mail_diff from_B_to_A .mail/ñangapirí@srv-ñ + +success diff --git a/test/util/lib.sh b/test/util/lib.sh index 8dedbf2..6c2924b 100644 --- a/test/util/lib.sh +++ b/test/util/lib.sh @@ -39,8 +39,11 @@ function chasquid() { } function add_user() { + CONFDIR="${CONFDIR:-config}" + mkdir -p "${CONFDIR}/domains/${1}/" go run ${TBASE}/../../cmd/chasquid-util/chasquid-util.go \ - adduser "config/domains/${1}/users" "${2}" --password "${3}" \ + adduser "${CONFDIR}/domains/${1}/users" "${2}" \ + --password "${3}" \ >> .add_user_logs } @@ -52,6 +55,10 @@ function run_msmtp() { msmtp -C msmtprc "$@" } +function smtpc.py() { + ${UTILDIR}/smtpc.py "$@" +} + function mail_diff() { ${UTILDIR}/mail_diff "$@" } @@ -82,9 +89,21 @@ function wait_for_file() { # Generate certs for the given hostname. function generate_certs_for() { - mkdir -p config/certs/${1}/ + CONFDIR="${CONFDIR:-config}" + mkdir -p ${CONFDIR}/certs/${1}/ ( - cd config/certs/${1} + cd ${CONFDIR}/certs/${1} generate_cert -ca -duration=1h -host=${1} ) } + +# Check the Python version, and skip if it's too old. +# This will check against the version required for smtpc.py. +function skip_if_python_is_too_old() { + # We need Python >= 3.5 to be able to use SMTPUTF8. + check='import sys; sys.exit(0 if sys.version_info >= (3, 5) else 1)' + if ! python3 -c "${check}"; then + skip "python3 >= 3.5 not available" + exit 0 + fi +} diff --git a/test/util/smtpc.py b/test/util/smtpc.py new file mode 100755 index 0000000..8d459c2 --- /dev/null +++ b/test/util/smtpc.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# +# Simple SMTP client for testing purposes. + +import argparse +import email.parser +import email.policy +import smtplib +import sys + +ap = argparse.ArgumentParser() +ap.add_argument("--server", help="SMTP server to connect to") +ap.add_argument("--user", help="Username to use in SMTP AUTH") +ap.add_argument("--password", help="Password to use in SMTP AUTH") +args = ap.parse_args() + +# Parse the email using the "default" policy, which is not really the default. +# If unspecified, compat32 is used, which does not support UTF8. +msg = email.parser.Parser(policy=email.policy.default).parse(sys.stdin) + +s = smtplib.SMTP(args.server) +s.starttls() +s.login(args.user, args.password) + +# Note this does NOT support non-ascii message payloads transparently (headers +# are ok). +s.send_message(msg) +s.quit() + +