courier: Support SMTPUTF8 in the SMTP courier

This patch adds SMTPUTF8 to the SMTP courier.

It introduces a new internal/smtp package that extends Go's net/smtp with
SMTPUTF8 (in a very narrow way, not for general use), and then makes the
courier use it.

Also use an IDNA-safe version when saying hello, otherwise servers could
complain if the hostname is not ASCII, and at that stage we don't know if they
support it or not.

internal/courier/smtp.go

@@ -4,13 +4,13 @@ import (
 	"crypto/tls"
 	"flag"
 	"net"
-	"net/smtp"
 	"time"
 
 	"github.com/golang/glog"
 	"golang.org/x/net/idna"
 
 	"blitiri.com.ar/go/chasquid/internal/envelope"
+	"blitiri.com.ar/go/chasquid/internal/smtp"
 	"blitiri.com.ar/go/chasquid/internal/trace"
 )
 
@@ -65,7 +65,11 @@ retry:
 
 	// Issue an EHLO with a valid domain; otherwise, some servers like postfix
 	// will complain.
-	if err = c.Hello(envelope.DomainOf(from)); err != nil {
+	fromDomain, err := idna.ToASCII(envelope.DomainOf(from))
+	if err != nil {
+		return tr.Errorf("Sender domain not IDNA compliant: %v", err), true
+	}
+	if err = c.Hello(fromDomain); err != nil {
 		return tr.Errorf("Error saying hello: %v", err), false
 	}
 
@@ -98,20 +102,12 @@ retry:
 		tr.LazyPrintf("Insecure - not using TLS")
 	}
 
-	// TODO: check if the errors we get back are transient or not.
-	// Go's smtp does not allow us to do this, so leave for when we do it
-	// ourselves.
-
 	// c.Mail will add the <> for us when the address is empty.
 	if from == "<>" {
 		from = ""
 	}
-	if err = c.Mail(from); err != nil {
-		return tr.Errorf("MAIL %v", err), false
-	}
-
-	if err = c.Rcpt(to); err != nil {
-		return tr.Errorf("RCPT TO %v", err), false
+	if err = c.MailAndRcpt(from, to); err != nil {
+		return tr.Errorf("MAIL+RCPT %v", err), false
 	}
 
 	w, err := c.Data()

internal/smtp/smtp.go

@@ -0,0 +1,129 @@
+// Package smtp implements the Simple Mail Transfer Protocol as defined in RFC
+// 5321.  It extends net/smtp as follows:
+//
+//  - Supports SMTPUTF8, via MailAndRcpt.
+//
+package smtp
+
+import (
+	"net"
+	"net/smtp"
+	"net/textproto"
+	"unicode"
+
+	"blitiri.com.ar/go/chasquid/internal/envelope"
+
+	"golang.org/x/net/idna"
+)
+
+// A Client represents a client connection to an SMTP server.
+type Client struct {
+	*smtp.Client
+}
+
+func NewClient(conn net.Conn, host string) (*Client, error) {
+	c, err := smtp.NewClient(conn, host)
+	if err != nil {
+		return nil, err
+	}
+	return &Client{c}, nil
+}
+
+// cmd sends a command and returns the response over the text connection.
+// Based on Go's method of the same name.
+func (c *Client) cmd(expectCode int, format string, args ...interface{}) (int, string, error) {
+	id, err := c.Text.Cmd(format, args...)
+	if err != nil {
+		return 0, "", err
+	}
+	c.Text.StartResponse(id)
+	defer c.Text.EndResponse(id)
+
+	return c.Text.ReadResponse(expectCode)
+}
+
+// MailAndRcpt issues MAIL FROM and RCPT TO commands, in sequence.
+// It will check the addresses, decide if SMTPUTF8 is needed, and apply the
+// necessary transformations.
+func (c *Client) MailAndRcpt(from string, to string) error {
+	from, from_needs, err := c.prepareForSMTPUTF8(from)
+	if err != nil {
+		return err
+	}
+
+	to, to_needs, err := c.prepareForSMTPUTF8(to)
+	if err != nil {
+		return err
+	}
+	smtputf8Needed := from_needs || to_needs
+
+	cmdStr := "MAIL FROM:<%s>"
+	if ok, _ := c.Extension("8BITMIME"); ok {
+		cmdStr += " BODY=8BITMIME"
+	}
+	if smtputf8Needed {
+		cmdStr += " SMTPUTF8"
+	}
+	_, _, err = c.cmd(250, cmdStr, from)
+	if err != nil {
+		return err
+	}
+
+	_, _, err = c.cmd(25, "RCPT TO:<%s>", to)
+	return err
+}
+
+// prepareForSMTPUTF8 prepares the address for SMTPUTF8.
+// It returns:
+//  - The address to use. It is based on addr, and possibly modified to make
+//    it not need the extension, if the server does not support it.
+//  - Whether the address needs the extension or not.
+//  - An error if the address needs the extension, but the client does not
+//    support it.
+func (c *Client) prepareForSMTPUTF8(addr string) (string, bool, error) {
+	// ASCII address pass through.
+	if isASCII(addr) {
+		return addr, false, nil
+	}
+
+	// Non-ASCII address also pass through if the server supports the
+	// extension.
+	// Note there's a chance the server wants the domain in IDNA anyway, but
+	// it could also require it to be UTF8. We assume that if it supports
+	// SMTPUTF8 then it knows what its doing.
+	if ok, _ := c.Extension("SMTPUTF8"); ok {
+		return addr, true, nil
+	}
+
+	// Something is not ASCII, and the server does not support SMTPUTF8:
+	//  - If it's the local part, there's no way out and is required.
+	//  - If it's the domain, use IDNA.
+	user, domain := envelope.Split(addr)
+
+	if !isASCII(user) {
+		return addr, true, &textproto.Error{599,
+			"local part is not ASCII but server does not support SMTPUTF8"}
+	}
+
+	// If it's only the domain, convert to IDNA and move on.
+	domain, err := idna.ToASCII(domain)
+	if err != nil {
+		// The domain is not IDNA compliant, which is odd.
+		// Fail with a permanent error, not ideal but this should not
+		// happen.
+		return addr, true, &textproto.Error{599,
+			"non-ASCII domain is not IDNA safe"}
+	}
+
+	return user + "@" + domain, false, nil
+}
+
+// isASCII returns true if all the characters in s are ASCII, false otherwise.
+func isASCII(s string) bool {
+	for _, c := range s {
+		if c > unicode.MaxASCII {
+			return false
+		}
+	}
+	return true
+}

internal/smtp/smtp_test.go

@@ -0,0 +1,186 @@
+package smtp
+
+import (
+	"bufio"
+	"bytes"
+	"net"
+	"net/smtp"
+	"net/textproto"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestIsASCII(t *testing.T) {
+	cases := []struct {
+		str   string
+		ascii bool
+	}{
+		{"", true},
+		{"<>", true},
+		{"lalala", true},
+		{"ñaca", false},
+		{"año", false},
+	}
+	for _, c := range cases {
+		if ascii := isASCII(c.str); ascii != c.ascii {
+			t.Errorf("%q: expected %v, got %v", c.str, c.ascii, ascii)
+		}
+	}
+}
+
+func TestBasic(t *testing.T) {
+	fake, client := fakeDialog(`> EHLO a_test
+< 250-server replies your hello
+< 250-SIZE 35651584
+< 250-SMTPUTF8
+< 250-8BITMIME
+< 250 HELP
+> MAIL FROM:<from@from> BODY=8BITMIME
+< 250 MAIL FROM is fine
+> RCPT TO:<to@to>
+< 250 RCPT TO is fine
+`)
+
+	c := &Client{
+		Client: &smtp.Client{Text: textproto.NewConn(fake)}}
+	if err := c.Hello("a_test"); err != nil {
+		t.Fatalf("Hello failed: %v", err)
+	}
+
+	if err := c.MailAndRcpt("from@from", "to@to"); err != nil {
+		t.Fatalf("MailAndRcpt failed: %v", err)
+	}
+
+	cmds := fake.Client()
+	if client != cmds {
+		t.Fatalf("Got:\n%s\nExpected:\n%s", cmds, client)
+	}
+}
+
+func TestSMTPUTF8(t *testing.T) {
+	fake, client := fakeDialog(`> EHLO araña
+< 250-chasquid replies your hello
+< 250-SIZE 35651584
+< 250-SMTPUTF8
+< 250-8BITMIME
+< 250 HELP
+> MAIL FROM:<año@ñudo> BODY=8BITMIME SMTPUTF8
+< 250 MAIL FROM is fine
+> RCPT TO:<ñaca@ñoño>
+< 250 RCPT TO is fine
+`)
+
+	c := &Client{
+		Client: &smtp.Client{Text: textproto.NewConn(fake)}}
+	if err := c.Hello("araña"); err != nil {
+		t.Fatalf("Hello failed: %v", err)
+	}
+
+	if err := c.MailAndRcpt("año@ñudo", "ñaca@ñoño"); err != nil {
+		t.Fatalf("MailAndRcpt failed: %v\nDialog: %s", err, fake.Client())
+	}
+
+	cmds := fake.Client()
+	if client != cmds {
+		t.Fatalf("Got:\n%s\nExpected:\n%s", cmds, client)
+	}
+}
+
+func TestSMTPUTF8NotSupported(t *testing.T) {
+	fake, client := fakeDialog(`> EHLO araña
+< 250-chasquid replies your hello
+< 250-SIZE 35651584
+< 250-8BITMIME
+< 250 HELP
+`)
+
+	c := &Client{
+		Client: &smtp.Client{Text: textproto.NewConn(fake)}}
+	if err := c.Hello("araña"); err != nil {
+		t.Fatalf("Hello failed: %v", err)
+	}
+
+	if err := c.MailAndRcpt("año@ñudo", "ñaca@ñoño"); err != nil {
+		terr, ok := err.(*textproto.Error)
+		if !ok || terr.Code != 599 {
+			t.Fatalf("MailAndRcpt failed with unexpected error: %v\nDialog: %s",
+				err, fake.Client())
+		}
+	}
+
+	cmds := fake.Client()
+	if client != cmds {
+		t.Fatalf("Got:\n%s\nExpected:\n%s", cmds, client)
+	}
+}
+
+func TestFallbackToIDNA(t *testing.T) {
+	fake, client := fakeDialog(`> EHLO araña
+< 250-chasquid replies your hello
+< 250-SIZE 35651584
+< 250-8BITMIME
+< 250 HELP
+> MAIL FROM:<gran@xn--udo-6ma> BODY=8BITMIME
+< 250 MAIL FROM is fine
+> RCPT TO:<alto@xn--oo-yjab>
+< 250 RCPT TO is fine
+`)
+
+	c := &Client{
+		Client: &smtp.Client{Text: textproto.NewConn(fake)}}
+	if err := c.Hello("araña"); err != nil {
+		t.Fatalf("Hello failed: %v", err)
+	}
+
+	if err := c.MailAndRcpt("gran@ñudo", "alto@ñoño"); err != nil {
+		terr, ok := err.(*textproto.Error)
+		if !ok || terr.Code != 599 {
+			t.Fatalf("MailAndRcpt failed with unexpected error: %v\nDialog: %s",
+				err, fake.Client())
+		}
+	}
+
+	cmds := fake.Client()
+	if client != cmds {
+		t.Fatalf("Got:\n%s\nExpected:\n%s", cmds, client)
+	}
+}
+
+type faker struct {
+	buf *bytes.Buffer
+	*bufio.ReadWriter
+}
+
+func (f faker) Close() error                     { return nil }
+func (f faker) LocalAddr() net.Addr              { return nil }
+func (f faker) RemoteAddr() net.Addr             { return nil }
+func (f faker) SetDeadline(time.Time) error      { return nil }
+func (f faker) SetReadDeadline(time.Time) error  { return nil }
+func (f faker) SetWriteDeadline(time.Time) error { return nil }
+func (f faker) Client() string {
+	f.ReadWriter.Writer.Flush()
+	return f.buf.String()
+}
+
+// Takes a dialog, returns the corresponding faker and expected client
+// messages.  Ideally we would check this interactively, and it's not that
+// difficult, but this is good enough for now.
+func fakeDialog(dialog string) (faker, string) {
+	var client, server string
+
+	for _, l := range strings.Split(dialog, "\n") {
+		if strings.HasPrefix(l, "< ") {
+			server += l[2:] + "\r\n"
+		} else if strings.HasPrefix(l, "> ") {
+			client += l[2:] + "\r\n"
+		}
+	}
+
+	fake := faker{}
+	fake.buf = &bytes.Buffer{}
+	fake.ReadWriter = bufio.NewReadWriter(
+		bufio.NewReader(strings.NewReader(server)), bufio.NewWriter(fake.buf))
+
+	return fake, client
+}