test: Add stress tests

This patch introduces stress tests, which put load on the daemon to help
sanity check its behaviour under stress.

They are separate from the existing integration tests, which focus on
correctness.

Two tests are included here: a load test, which sends emails repeatedly;
and a connection test which opens as many conections as possible.

Makefile

@@ -39,6 +39,7 @@ dovecot-auth-cli:
 test:
 	go test ${GOFLAGS} ./...
 	setsid -w ./test/run.sh
+	setsid -w ./test/stress.sh
 	setsid -w ./cmd/chasquid-util/test.sh
 	setsid -w ./cmd/mda-lmtp/test.sh
 	setsid -w ./cmd/dovecot-auth-cli/test.sh

test/.gitignore

@@ -2,4 +2,6 @@
 # Ignore the user databases - we create them each time.
 t-*/config/**/users
 t-*/?/**/users
+stress-*/config/**/users
+stress-*/?/**/users
 

test/stress-01-load/config/chasquid.conf

@@ -0,0 +1,15 @@
+hostname: "testserver"
+
+smtp_address: ":1025"
+submission_address: ":1587"
+submission_over_tls_address: ":1465"
+monitoring_address: ":1099"
+
+mail_delivery_agent_bin: "test-mda"
+mail_delivery_agent_args: "%to%"
+
+data_dir: "../.data"
+mail_log_path: "../.logs/mail_log"
+
+suffix_separators: "+-"
+drop_characters: "._"

test/stress-01-load/config/domains/testserver/aliases

@@ -0,0 +1,4 @@
+
+null: | true
+fail: | false
+

test/stress-01-load/hosts

@@ -0,0 +1 @@
+testserver localhost

test/stress-01-load/run.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+. $(dirname ${0})/../util/lib.sh
+
+init
+
+generate_certs_for testserver
+add_user user@testserver secretpassword
+
+# Note we run the server with minimal logging, to avoid generating very large
+# log files, which are not very useful anyway.
+mkdir -p .logs
+chasquid -v=-1 --logfile=.logs/chasquid.log --config_dir=config &
+wait_until_ready 1025
+
+echo Peak RAM: `chasquid_ram_peak`
+
+if ! loadgen -logtime -addr=localhost:1025 -run_for=3s -noop; then
+	fail
+fi
+
+echo Peak RAM: `chasquid_ram_peak`
+
+if ! loadgen -logtime -addr=localhost:1025 -run_for=3s; then
+	fail
+fi
+
+echo Peak RAM: `chasquid_ram_peak`
+
+success

test/stress-02-connections/config/chasquid.conf

@@ -0,0 +1,15 @@
+hostname: "testserver"
+
+smtp_address: ":1025"
+submission_address: ":1587"
+submission_over_tls_address: ":1465"
+monitoring_address: ":1099"
+
+mail_delivery_agent_bin: "test-mda"
+mail_delivery_agent_args: "%to%"
+
+data_dir: "../.data"
+mail_log_path: "../.logs/mail_log"
+
+suffix_separators: "+-"
+drop_characters: "._"

test/stress-02-connections/hosts

@@ -0,0 +1 @@
+testserver localhost

test/stress-02-connections/run.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+. $(dirname ${0})/../util/lib.sh
+
+init
+
+generate_certs_for testserver
+add_user user@testserver secretpassword
+
+# Note we run the server with minimal logging, to avoid generating very large
+# log files, which are not very useful anyway.
+mkdir -p .logs
+chasquid -v=-1 --logfile=.logs/chasquid.log --config_dir=config &
+wait_until_ready 1025
+
+echo Peak RAM: `chasquid_ram_peak`
+
+# Set connection count to (max open files) - (leeway).
+# We set the leeway to account for file descriptors opened by the runtime and
+# listeners; 20 should be enough for now.
+# Cap it to 2000, as otherwise it can be problematic due to port availability.
+COUNT=$(( `ulimit -n` - 20 ))
+if [ $COUNT -gt 2000 ]; then
+	COUNT=2000
+fi
+
+if ! conngen -logtime -addr=localhost:1025 -count=$COUNT; then
+	tail -n 1 .logs/chasquid.log
+	fail
+fi
+
+echo Peak RAM: `chasquid_ram_peak`
+
+success

test/stress.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+. $(dirname ${0})/util/lib.sh
+
+init
+
+FAILED=0
+
+for i in stress-*; do
+	echo $i ...
+	setsid -w $i/run.sh
+	FAILED=$(( $FAILED + $? ))
+	echo
+done
+
+exit $FAILED

test/util/conngen.go

@@ -0,0 +1,109 @@
+// +build ignore
+
+// SMTP connection generator, for testing purposes.
+package main
+
+import (
+	"flag"
+	"net"
+	"net/http"
+	"net/smtp"
+	"time"
+
+	"golang.org/x/net/trace"
+
+	_ "net/http/pprof"
+
+	"blitiri.com.ar/go/log"
+)
+
+var (
+	addr = flag.String("addr", "",
+		"server address")
+	httpAddr = flag.String("http_addr", "localhost:8011",
+		"monitoring HTTP server listening address")
+	wait = flag.Bool("wait", false,
+		"don't exit after --run_for has lapsed")
+	count = flag.Int("count", 1000,
+		"how many connections to open")
+)
+
+var (
+	host string
+	exit bool
+)
+
+func main() {
+	var err error
+
+	flag.Parse()
+	log.Init()
+
+	host, _, err = net.SplitHostPort(*addr)
+	if err != nil {
+		log.Fatalf("failed to split --addr=%q: %v", *addr, err)
+	}
+
+	if *wait {
+		go http.ListenAndServe(*httpAddr, nil)
+		log.Infof("monitoring address: http://%v/debug/requests?fam=one&b=11",
+			*httpAddr)
+	}
+
+	log.Infof("creating %d simultaneous connections", *count)
+	conns := []*C{}
+	for i := 0; i < *count; i++ {
+		c, err := newC()
+		if err != nil {
+			log.Fatalf("failed to connect #%d: %v", i, err)
+		}
+
+		conns = append(conns, c)
+
+		if i%200 == 0 {
+			log.Infof("  ... %d connections", i)
+
+		}
+	}
+	log.Infof("done, created %d simultaneous connections", *count)
+
+	if *wait {
+		for {
+			time.Sleep(24 * time.Hour)
+		}
+	}
+}
+
+type C struct {
+	tr trace.Trace
+	n  net.Conn
+	s  *smtp.Client
+}
+
+func newC() (*C, error) {
+	tr := trace.New("conn", *addr)
+
+	conn, err := net.Dial("tcp", *addr)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := smtp.NewClient(conn, host)
+	if err != nil {
+		conn.Close()
+		return nil, err
+	}
+
+	err = client.Hello(host)
+	if err != nil {
+		return nil, err
+	}
+
+	return &C{tr: tr, n: conn, s: client}, nil
+}
+
+func (c *C) close() {
+	c.tr.Finish()
+	c.s.Close()
+	c.n.Close()
+}

test/util/lib.sh

@@ -24,23 +24,21 @@ function init() {
 	trap "kill 0" EXIT
 }
 
-function generate_cert() {
-	go run ${UTILDIR}/generate_cert.go "$@"
-}
-
 function chasquid() {
 	if [ "${COVER_DIR}" != "" ]; then
 		chasquid_cover "$@"
 		return
 	fi
 
+	( cd ${TBASE}/../../; go build ${RACE} . )
+
 	# HOSTALIASES: so we "fake" hostnames.
 	# PATH: so chasquid can call test-mda without path issues.
 	# MDA_DIR: so our test-mda knows where to deliver emails.
 	HOSTALIASES=${TBASE}/hosts \
 	PATH=${UTILDIR}:${PATH} \
 	MDA_DIR=${TBASE}/.mail \
-		go run ${RACE} ${TBASE}/../../chasquid.go "$@"
+		${TBASE}/../../chasquid "$@"
 }
 
 function chasquid_cover() {
@@ -96,6 +94,18 @@ function chamuyero() {
 	${UTILDIR}/chamuyero "$@"
 }
 
+function generate_cert() {
+	go run ${UTILDIR}/generate_cert.go "$@"
+}
+
+function loadgen() {
+	go run ${UTILDIR}/loadgen.go "$@"
+}
+
+function conngen() {
+	go run ${UTILDIR}/conngen.go "$@"
+}
+
 function success() {
 	echo success
 }
@@ -145,3 +155,11 @@ function skip_if_python_is_too_old() {
 		skip "python3 >= 3.5 not available"
 	fi
 }
+
+function chasquid_ram_peak() {
+	# Find the pid of the daemon, which we expect is running on the
+	# background somewhere within our current session.
+	SERVER_PID=`pgrep -s 0 -x chasquid`
+
+	echo $( cat /proc/$SERVER_PID/status | grep VmHWM | cut -d ':' -f 2- )
+}

test/util/loadgen.go

@@ -0,0 +1,195 @@
+// +build ignore
+
+// SMTP load generator, for testing purposes.
+package main
+
+import (
+	"flag"
+	"net"
+	"net/http"
+	"runtime"
+	"sync"
+	"time"
+
+	_ "net/http/pprof"
+
+	"golang.org/x/net/trace"
+
+	"blitiri.com.ar/go/chasquid/internal/smtp"
+	"blitiri.com.ar/go/log"
+)
+
+var (
+	addr = flag.String("addr", "",
+		"server address")
+	httpAddr = flag.String("http_addr", "localhost:8011",
+		"monitoring HTTP server listening address")
+	parallel = flag.Int("parallel", 0,
+		"how many sending loops to run in parallel")
+	runFor = flag.Duration("run_for", 0,
+		"how long to run for (0 = forever)")
+	wait = flag.Bool("wait", false,
+		"don't exit after --run_for has lapsed")
+	noop = flag.Bool("noop", false,
+		"don't send an email, just connect and run a NOOP")
+)
+
+var (
+	host string
+	exit bool
+
+	globalCount   int64 = 0
+	globalRuntime time.Duration
+	globalMu      = &sync.Mutex{}
+)
+
+func main() {
+	var err error
+
+	flag.Parse()
+	log.Init()
+
+	host, _, err = net.SplitHostPort(*addr)
+	if err != nil {
+		log.Fatalf("failed to split --addr=%q: %v", *addr, err)
+	}
+
+	if *wait {
+		go http.ListenAndServe(*httpAddr, nil)
+		log.Infof("monitoring address: http://%v/debug/requests?fam=one&b=11",
+			*httpAddr)
+	}
+
+	if *parallel == 0 {
+		*parallel = runtime.GOMAXPROCS(0)
+	}
+
+	lt := "full"
+	if *noop {
+		lt = "noop"
+	}
+
+	log.Infof("launching %d %s sending loops in parallel", *parallel, lt)
+	for i := 0; i < *parallel; i++ {
+		go serial(i)
+	}
+
+	var totalCount int64
+	var totalRuntime time.Duration
+	start := time.Now()
+	for range time.Tick(1 * time.Second) {
+		globalMu.Lock()
+		totalCount += globalCount
+		totalRuntime += globalRuntime
+		count := globalCount
+		runtime := globalRuntime
+		globalCount = 0
+		globalRuntime = 0
+		globalMu.Unlock()
+
+		if count == 0 {
+			log.Infof("0 ops")
+		} else {
+			log.Infof("%d ops, %v /op", count,
+				time.Duration(runtime.Nanoseconds()/count).Truncate(time.Microsecond))
+		}
+
+		if *runFor > 0 && time.Since(start) > *runFor {
+			exit = true
+			break
+		}
+	}
+
+	end := time.Now()
+	window := end.Sub(start)
+	log.Infof("total: %d ops, %v wall, %v run",
+		totalCount,
+		window.Truncate(time.Millisecond),
+		totalRuntime.Truncate(time.Millisecond))
+
+	avgLat := time.Duration(totalRuntime.Nanoseconds() / totalCount)
+	log.Infof("avg: %v /op, %.0f ops/s",
+		avgLat.Truncate(time.Microsecond),
+		float64(totalCount)/window.Seconds(),
+	)
+
+	if *wait {
+		for {
+			time.Sleep(24 * time.Hour)
+		}
+	}
+}
+
+func serial(id int) {
+	var count int64
+	start := time.Now()
+	for {
+		count += 1
+		err := one()
+		if err != nil {
+			log.Fatalf("%v", err)
+		}
+
+		if count == 5 {
+			globalMu.Lock()
+			globalCount += count
+			globalRuntime += time.Since(start)
+			globalMu.Unlock()
+			count = 0
+			start = time.Now()
+
+			if exit {
+				return
+			}
+		}
+	}
+}
+
+func one() error {
+	tr := trace.New("one", *addr)
+	defer tr.Finish()
+
+	conn, err := net.Dial("tcp", *addr)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	client, err := smtp.NewClient(conn, host)
+	if err != nil {
+		return err
+	}
+	defer client.Close()
+
+	if *noop {
+		err = client.Noop()
+		if err != nil {
+			return err
+		}
+	} else {
+		err = client.MailAndRcpt("test@test", "null@testserver")
+		if err != nil {
+			return err
+		}
+
+		w, err := client.Data()
+		if err != nil {
+			return err
+		}
+		_, err = w.Write(body)
+		if err != nil {
+			return err
+		}
+		err = w.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+var body = []byte(`Subject: Load test
+
+This is the body of the load test email.
+`)