smtpsrv: Implement a post-DATA hook

This patch implements a post-DATA hook, which is run after receiving the
data but before sending a reply.

It can be used to implement content filtering when receiving email, for
example for passing the email through an anti-spam or an anti-virus.

chasquid.go

@@ -62,6 +62,7 @@ func main() {
 	s := smtpsrv.NewServer()
 	s.Hostname = conf.Hostname
 	s.MaxDataSize = conf.MaxDataSizeMb * 1024 * 1024
+	s.PostDataHook = "hooks/post-data"
 
 	s.SetAliasesConfig(conf.SuffixSeparators, conf.DropCharacters)
 

hooks/post-data

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# This file is an example post-data hook that will run standard filtering
+# utilities if they are available.
+#
+#  - spamc (from Spamassassin) to filter spam.
+#  - clamdscan (from ClamAV) to filter virus.
+
+set -e
+
+TF="$(mktemp --tmpdir "post-data-XXXXXXXXXX")"
+trap 'rm "$TF"' EXIT
+
+# Save the message to the temporary file, so we can pass it on to the various
+# filters.
+cat > "$TF"
+
+
+if command -v spamc >/dev/null; then
+        if ! SL=$(spamc -c - < "$TF") ; then
+                echo "spam detected"
+                exit 1
+        fi
+        echo "X-Spam-Score: $SL"
+fi
+
+
+if command -v clamdscan >/dev/null; then
+        if ! clamdscan --no-summary --infected - < "$TF" 1>&2 ; then
+                echo "virus detected"
+                exit 1
+        fi
+        echo "X-Virus-Scanned: pass"
+fi
+

internal/smtpsrv/conn.go

@@ -2,6 +2,7 @@ package smtpsrv
 
 import (
 	"bytes"
+	"context"
 	"crypto/tls"
 	"expvar"
 	"fmt"
@@ -11,6 +12,8 @@ import (
 	"net"
 	"net/mail"
 	"net/textproto"
+	"os"
+	"os/exec"
 	"strconv"
 	"strings"
 	"time"
@@ -36,6 +39,7 @@ var (
 	loopsDetected     = expvar.NewInt("chasquid/smtpIn/loopsDetected")
 	tlsCount          = expvar.NewMap("chasquid/smtpIn/tlsCount")
 	slcResults        = expvar.NewMap("chasquid/smtpIn/securityLevelChecks")
+	hookResults       = expvar.NewMap("chasquid/smtpIn/hookResults")
 )
 
 // Global event logs.
@@ -61,6 +65,9 @@ type Conn struct {
 	// Maximum data size.
 	maxDataSize int64
 
+	// Post-DATA hook location.
+	postDataHook string
+
 	// Connection information.
 	conn         net.Conn
 	tc           *textproto.Conn
@@ -514,6 +521,12 @@ func (c *Conn) DATA(params string) (code int, msg string) {
 
 	c.addReceivedHeader()
 
+	hookOut, err := c.runPostDataHook(c.data)
+	if err != nil {
+		return 554, err.Error()
+	}
+	c.data = append(hookOut, c.data...)
+
 	// There are no partial failures here: we put it in the queue, and then if
 	// individual deliveries fail, we report via email.
 	msgID, err := c.queue.Put(c.mailFrom, c.rcptTo, c.data)
@@ -599,6 +612,115 @@ func checkData(data []byte) error {
 	return nil
 }
 
+// runPostDataHook and return the new headers to add, an error (if any), and
+// true if the error is permanent or false if transient.
+func (c *Conn) runPostDataHook(data []byte) ([]byte, error) {
+	// TODO: check if the file is executable.
+	if _, err := os.Stat(c.postDataHook); os.IsNotExist(err) {
+		hookResults.Add("post-data:skip", 1)
+		return nil, nil
+	}
+	tr := trace.New("Hook.Post-DATA", c.conn.RemoteAddr().String())
+	defer tr.Finish()
+	tr.Debugf("running")
+
+	ctx, cancel := context.WithDeadline(context.Background(),
+		time.Now().Add(1*time.Minute))
+	defer cancel()
+	cmd := exec.CommandContext(ctx, c.postDataHook)
+	cmd.Stdin = bytes.NewReader(data)
+
+	// Prepare the environment, copying some common variables so the hook has
+	// someting reasonable, and then setting the specific ones for this case.
+	for _, v := range strings.Fields("USER PWD SHELL PATH") {
+		cmd.Env = append(cmd.Env, v+"="+os.Getenv(v))
+	}
+	cmd.Env = append(cmd.Env, "REMOTE_ADDR="+c.conn.RemoteAddr().String())
+	cmd.Env = append(cmd.Env, "MAIL_FROM="+c.mailFrom)
+	cmd.Env = append(cmd.Env, "RCPT_TO="+strings.Join(c.rcptTo, " "))
+	cmd.Env = append(cmd.Env, "AUTH_AS="+c.authUser+"@"+c.authDomain)
+	if c.onTLS {
+		cmd.Env = append(cmd.Env, "ON_TLS=1")
+	}
+	if envelope.DomainIn(c.mailFrom, c.localDomains) {
+		cmd.Env = append(cmd.Env, "FROM_LOCAL_DOMAIN=1")
+	}
+
+	out, err := cmd.Output()
+	if err != nil {
+		hookResults.Add("post-data:fail", 1)
+		tr.Error(err)
+		tr.Debugf("stdout: %s", out)
+		if ee, ok := err.(*exec.ExitError); ok {
+			tr.Printf("stderr: %s", string(ee.Stderr))
+		}
+
+		// The error contains the last line of stdout, so filters can pass
+		// some rejection information back to the sender.
+		err = fmt.Errorf(lastLine(string(out)))
+		return nil, err
+	}
+
+	// Check that output looks like headers, to avoid breaking the email
+	// contents. If it does not, just skip it.
+	if !isHeader(out) {
+		hookResults.Add("post-data:badoutput", 1)
+		tr.Errorf("error parsing post-data output: '%s'", out)
+		return nil, nil
+	}
+
+	tr.Debugf("success")
+	tr.Debugf("stdout: %s", out)
+	hookResults.Add("post-data:success", 1)
+	return out, nil
+}
+
+// isHeader checks if the given buffer is a valid MIME header.
+func isHeader(b []byte) bool {
+	s := string(b)
+	if len(s) == 0 {
+		return true
+	}
+
+	// If it is just a \n, or contains two \n, then it's not a header.
+	if s == "\n" || strings.Contains(s, "\n\n") {
+		return false
+	}
+
+	// If it does not end in \n, not a header.
+	if s[len(s)-1] != '\n' {
+		return false
+	}
+
+	// Each line must either start with a space or have a ':'.
+	seen := false
+	for _, line := range strings.SplitAfter(s, "\n") {
+		if line == "" {
+			continue
+		}
+		if strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") {
+			if !seen {
+				// Continuation without a header first (invalid).
+				return false
+			}
+			continue
+		}
+		if !strings.Contains(line, ":") {
+			return false
+		}
+		seen = true
+	}
+	return true
+}
+
+func lastLine(s string) string {
+	l := strings.Split(s, "\n")
+	if len(l) < 2 {
+		return ""
+	}
+	return l[len(l)-2]
+}
+
 func (c *Conn) STARTTLS(params string) (code int, msg string) {
 	if c.onTLS {
 		return 503, "You are already wearing that!"

internal/smtpsrv/conn_test.go

@@ -58,3 +58,26 @@ func TestSecLevel(t *testing.T) {
 		t.Fatalf("plain seclevel worked, downgrade was allowed")
 	}
 }
+
+func TestIsHeader(t *testing.T) {
+	no := []string{
+		"a", "\n", "\n\n", " \n", " ",
+		"a:b", "a:  b\nx: y",
+		"\na:b\n", " a\nb:c\n",
+	}
+	for _, s := range no {
+		if isHeader([]byte(s)) {
+			t.Errorf("%q accepted as header, should be rejected", s)
+		}
+	}
+
+	yes := []string{
+		"", "a:b\n",
+		"X-Post-Data: success\n",
+	}
+	for _, s := range yes {
+		if !isHeader([]byte(s)) {
+			t.Errorf("%q rejected as header, should be accepted", s)
+		}
+	}
+}

internal/smtpsrv/server.go

@@ -53,6 +53,9 @@ type Server struct {
 
 	// Queue where we put incoming mail.
 	queue *queue.Queue
+
+	// Path to the Post-DATA hook.
+	PostDataHook string
 }
 
 func NewServer() *Server {
@@ -193,6 +196,7 @@ func (s *Server) serve(l net.Listener, mode SocketMode) {
 		sc := &Conn{
 			hostname:       s.Hostname,
 			maxDataSize:    s.MaxDataSize,
+			postDataHook:   s.PostDataHook,
 			conn:           conn,
 			tc:             textproto.NewConn(conn),
 			mode:           mode,

test/t-10-hooks/.gitignore

@@ -0,0 +1 @@
+config/hooks/post-data

test/t-10-hooks/config/chasquid.conf

@@ -0,0 +1,8 @@
+smtp_address: ":1025"
+submission_address: ":1587"
+monitoring_address: ":1099"
+
+mail_delivery_agent_bin: "test-mda"
+mail_delivery_agent_args: "%to%"
+
+data_dir: "../.data"

test/t-10-hooks/config/hooks/post-data.bad1

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+echo $0 > ../.data/post-data.out
+echo "This is not a header"
+

test/t-10-hooks/config/hooks/post-data.bad2

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+echo $0 > ../.data/post-data.out
+
+echo "X-Post-DATA: This starts like a header"
+echo
+echo "But then is not"
+

test/t-10-hooks/config/hooks/post-data.bad3

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo $0 > ../.data/post-data.out
+
+# Just a newline is quite problematic, as it would break the headers.
+echo
+

test/t-10-hooks/config/hooks/post-data.bad4

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+echo $0 > ../.data/post-data.out
+
+echo -n "X-Post-DATA: valid header with no newline at the end"

test/t-10-hooks/config/hooks/post-data.good

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+env > ../.data/post-data.out
+echo >> ../.data/post-data.out
+
+cat >> ../.data/post-data.out
+
+if [ "$RCPT_TO" == "blockme@testserver" ]; then
+	echo "¡No pasarán!"
+	exit 1
+fi
+
+echo "X-Post-Data: success"
+

test/t-10-hooks/content

@@ -0,0 +1,4 @@
+Subject: Prueba desde el test
+
+Crece desde el test el futuro
+Crece desde el test

test/t-10-hooks/hosts

@@ -0,0 +1 @@
+testserver localhost

test/t-10-hooks/msmtprc

@@ -0,0 +1,14 @@
+account default
+
+host testserver
+port 1587
+
+tls on
+tls_trust_file config/certs/testserver/fullchain.pem
+
+from user@testserver
+
+auth on
+user user@testserver
+password secretpassword
+

test/t-10-hooks/run.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -e
+. $(dirname ${0})/../util/lib.sh
+
+init
+
+generate_certs_for testserver
+add_user testserver user secretpassword
+add_user testserver someone secretpassword
+add_user testserver blockme secretpassword
+
+mkdir -p .logs
+chasquid -v=2 --log_dir=.logs --config_dir=config &
+wait_until_ready 1025
+
+cp config/hooks/post-data.good config/hooks/post-data
+
+run_msmtp someone@testserver < content
+
+wait_for_file .mail/someone@testserver
+
+mail_diff content .mail/someone@testserver
+
+if ! grep -q "X-Post-Data: success" .mail/someone@testserver; then
+	echo "missing X-Post-Data header"
+	exit 1
+fi
+
+function check() {
+	if ! grep -q "$1" .data/post-data.out; then
+		echo missing: $1
+		exit 1
+	fi
+}
+
+# Verify that the environment for the hook was reasonable.
+check "RCPT_TO=someone@testserver"
+check "MAIL_FROM=user@testserver"
+check "USER=$USER"
+check "PWD=$PWD/config"
+check "FROM_LOCAL_DOMAIN=1"
+check "ON_TLS=1"
+check "AUTH_AS=user@testserver"
+check "PATH="
+check "REMOTE_ADDR="
+
+
+# Check that a failure in the script results in failing delivery.
+if run_msmtp blockme@testserver < content 2>/dev/null; then
+	echo "ERROR: hook did not block email as expected"
+	exit 1
+fi
+
+# Check that the bad hooks don't prevent delivery.
+for i in config/hooks/post-data.bad*; do
+	cp $i config/hooks/post-data
+
+	run_msmtp someone@testserver < content
+	wait_for_file .mail/someone@testserver
+	mail_diff content .mail/someone@testserver
+done
+
+success