From b0011f5a5152b8017dce59c48362086bc8b6f6b6 Mon Sep 17 00:00:00 2001 From: Alberto Bertogli Date: Sun, 25 Mar 2018 13:33:16 +0100 Subject: [PATCH] test: Add stress tests This patch introduces stress tests, which put load on the daemon to help sanity check its behaviour under stress. They are separate from the existing integration tests, which focus on correctness. Two tests are included here: a load test, which sends emails repeatedly; and a connection test which opens as many conections as possible. --- Makefile | 1 + test/.gitignore | 2 + test/stress-01-load/config/chasquid.conf | 15 ++ .../config/domains/testserver/aliases | 4 + test/stress-01-load/hosts | 1 + test/stress-01-load/run.sh | 31 +++ .../config/chasquid.conf | 15 ++ test/stress-02-connections/hosts | 1 + test/stress-02-connections/run.sh | 35 ++++ test/stress.sh | 17 ++ test/util/conngen.go | 109 ++++++++++ test/util/lib.sh | 28 ++- test/util/loadgen.go | 195 ++++++++++++++++++ 13 files changed, 449 insertions(+), 5 deletions(-) create mode 100644 test/stress-01-load/config/chasquid.conf create mode 100644 test/stress-01-load/config/domains/testserver/aliases create mode 100644 test/stress-01-load/hosts create mode 100755 test/stress-01-load/run.sh create mode 100644 test/stress-02-connections/config/chasquid.conf create mode 100644 test/stress-02-connections/hosts create mode 100755 test/stress-02-connections/run.sh create mode 100755 test/stress.sh create mode 100644 test/util/conngen.go create mode 100644 test/util/loadgen.go diff --git a/Makefile b/Makefile index 17463aa..aa779ba 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,7 @@ dovecot-auth-cli: test: go test ${GOFLAGS} ./... setsid -w ./test/run.sh + setsid -w ./test/stress.sh setsid -w ./cmd/chasquid-util/test.sh setsid -w ./cmd/mda-lmtp/test.sh setsid -w ./cmd/dovecot-auth-cli/test.sh diff --git a/test/.gitignore b/test/.gitignore index 8495b0c..3f21709 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -2,4 +2,6 @@ # Ignore the user databases - we create them each time. t-*/config/**/users t-*/?/**/users +stress-*/config/**/users +stress-*/?/**/users diff --git a/test/stress-01-load/config/chasquid.conf b/test/stress-01-load/config/chasquid.conf new file mode 100644 index 0000000..9a35641 --- /dev/null +++ b/test/stress-01-load/config/chasquid.conf @@ -0,0 +1,15 @@ +hostname: "testserver" + +smtp_address: ":1025" +submission_address: ":1587" +submission_over_tls_address: ":1465" +monitoring_address: ":1099" + +mail_delivery_agent_bin: "test-mda" +mail_delivery_agent_args: "%to%" + +data_dir: "../.data" +mail_log_path: "../.logs/mail_log" + +suffix_separators: "+-" +drop_characters: "._" diff --git a/test/stress-01-load/config/domains/testserver/aliases b/test/stress-01-load/config/domains/testserver/aliases new file mode 100644 index 0000000..c24415f --- /dev/null +++ b/test/stress-01-load/config/domains/testserver/aliases @@ -0,0 +1,4 @@ + +null: | true +fail: | false + diff --git a/test/stress-01-load/hosts b/test/stress-01-load/hosts new file mode 100644 index 0000000..2b9b623 --- /dev/null +++ b/test/stress-01-load/hosts @@ -0,0 +1 @@ +testserver localhost diff --git a/test/stress-01-load/run.sh b/test/stress-01-load/run.sh new file mode 100755 index 0000000..1b2600e --- /dev/null +++ b/test/stress-01-load/run.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -e +. $(dirname ${0})/../util/lib.sh + +init + +generate_certs_for testserver +add_user user@testserver secretpassword + +# Note we run the server with minimal logging, to avoid generating very large +# log files, which are not very useful anyway. +mkdir -p .logs +chasquid -v=-1 --logfile=.logs/chasquid.log --config_dir=config & +wait_until_ready 1025 + +echo Peak RAM: `chasquid_ram_peak` + +if ! loadgen -logtime -addr=localhost:1025 -run_for=3s -noop; then + fail +fi + +echo Peak RAM: `chasquid_ram_peak` + +if ! loadgen -logtime -addr=localhost:1025 -run_for=3s; then + fail +fi + +echo Peak RAM: `chasquid_ram_peak` + +success diff --git a/test/stress-02-connections/config/chasquid.conf b/test/stress-02-connections/config/chasquid.conf new file mode 100644 index 0000000..9a35641 --- /dev/null +++ b/test/stress-02-connections/config/chasquid.conf @@ -0,0 +1,15 @@ +hostname: "testserver" + +smtp_address: ":1025" +submission_address: ":1587" +submission_over_tls_address: ":1465" +monitoring_address: ":1099" + +mail_delivery_agent_bin: "test-mda" +mail_delivery_agent_args: "%to%" + +data_dir: "../.data" +mail_log_path: "../.logs/mail_log" + +suffix_separators: "+-" +drop_characters: "._" diff --git a/test/stress-02-connections/hosts b/test/stress-02-connections/hosts new file mode 100644 index 0000000..2b9b623 --- /dev/null +++ b/test/stress-02-connections/hosts @@ -0,0 +1 @@ +testserver localhost diff --git a/test/stress-02-connections/run.sh b/test/stress-02-connections/run.sh new file mode 100755 index 0000000..1fb4bbd --- /dev/null +++ b/test/stress-02-connections/run.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e +. $(dirname ${0})/../util/lib.sh + +init + +generate_certs_for testserver +add_user user@testserver secretpassword + +# Note we run the server with minimal logging, to avoid generating very large +# log files, which are not very useful anyway. +mkdir -p .logs +chasquid -v=-1 --logfile=.logs/chasquid.log --config_dir=config & +wait_until_ready 1025 + +echo Peak RAM: `chasquid_ram_peak` + +# Set connection count to (max open files) - (leeway). +# We set the leeway to account for file descriptors opened by the runtime and +# listeners; 20 should be enough for now. +# Cap it to 2000, as otherwise it can be problematic due to port availability. +COUNT=$(( `ulimit -n` - 20 )) +if [ $COUNT -gt 2000 ]; then + COUNT=2000 +fi + +if ! conngen -logtime -addr=localhost:1025 -count=$COUNT; then + tail -n 1 .logs/chasquid.log + fail +fi + +echo Peak RAM: `chasquid_ram_peak` + +success diff --git a/test/stress.sh b/test/stress.sh new file mode 100755 index 0000000..1a8016f --- /dev/null +++ b/test/stress.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e +. $(dirname ${0})/util/lib.sh + +init + +FAILED=0 + +for i in stress-*; do + echo $i ... + setsid -w $i/run.sh + FAILED=$(( $FAILED + $? )) + echo +done + +exit $FAILED diff --git a/test/util/conngen.go b/test/util/conngen.go new file mode 100644 index 0000000..8a98979 --- /dev/null +++ b/test/util/conngen.go @@ -0,0 +1,109 @@ +// +build ignore + +// SMTP connection generator, for testing purposes. +package main + +import ( + "flag" + "net" + "net/http" + "net/smtp" + "time" + + "golang.org/x/net/trace" + + _ "net/http/pprof" + + "blitiri.com.ar/go/log" +) + +var ( + addr = flag.String("addr", "", + "server address") + httpAddr = flag.String("http_addr", "localhost:8011", + "monitoring HTTP server listening address") + wait = flag.Bool("wait", false, + "don't exit after --run_for has lapsed") + count = flag.Int("count", 1000, + "how many connections to open") +) + +var ( + host string + exit bool +) + +func main() { + var err error + + flag.Parse() + log.Init() + + host, _, err = net.SplitHostPort(*addr) + if err != nil { + log.Fatalf("failed to split --addr=%q: %v", *addr, err) + } + + if *wait { + go http.ListenAndServe(*httpAddr, nil) + log.Infof("monitoring address: http://%v/debug/requests?fam=one&b=11", + *httpAddr) + } + + log.Infof("creating %d simultaneous connections", *count) + conns := []*C{} + for i := 0; i < *count; i++ { + c, err := newC() + if err != nil { + log.Fatalf("failed to connect #%d: %v", i, err) + } + + conns = append(conns, c) + + if i%200 == 0 { + log.Infof(" ... %d connections", i) + + } + } + log.Infof("done, created %d simultaneous connections", *count) + + if *wait { + for { + time.Sleep(24 * time.Hour) + } + } +} + +type C struct { + tr trace.Trace + n net.Conn + s *smtp.Client +} + +func newC() (*C, error) { + tr := trace.New("conn", *addr) + + conn, err := net.Dial("tcp", *addr) + if err != nil { + return nil, err + } + + client, err := smtp.NewClient(conn, host) + if err != nil { + conn.Close() + return nil, err + } + + err = client.Hello(host) + if err != nil { + return nil, err + } + + return &C{tr: tr, n: conn, s: client}, nil +} + +func (c *C) close() { + c.tr.Finish() + c.s.Close() + c.n.Close() +} diff --git a/test/util/lib.sh b/test/util/lib.sh index 563424b..4fe2ece 100644 --- a/test/util/lib.sh +++ b/test/util/lib.sh @@ -24,23 +24,21 @@ function init() { trap "kill 0" EXIT } -function generate_cert() { - go run ${UTILDIR}/generate_cert.go "$@" -} - function chasquid() { if [ "${COVER_DIR}" != "" ]; then chasquid_cover "$@" return fi + ( cd ${TBASE}/../../; go build ${RACE} . ) + # HOSTALIASES: so we "fake" hostnames. # PATH: so chasquid can call test-mda without path issues. # MDA_DIR: so our test-mda knows where to deliver emails. HOSTALIASES=${TBASE}/hosts \ PATH=${UTILDIR}:${PATH} \ MDA_DIR=${TBASE}/.mail \ - go run ${RACE} ${TBASE}/../../chasquid.go "$@" + ${TBASE}/../../chasquid "$@" } function chasquid_cover() { @@ -96,6 +94,18 @@ function chamuyero() { ${UTILDIR}/chamuyero "$@" } +function generate_cert() { + go run ${UTILDIR}/generate_cert.go "$@" +} + +function loadgen() { + go run ${UTILDIR}/loadgen.go "$@" +} + +function conngen() { + go run ${UTILDIR}/conngen.go "$@" +} + function success() { echo success } @@ -145,3 +155,11 @@ function skip_if_python_is_too_old() { skip "python3 >= 3.5 not available" fi } + +function chasquid_ram_peak() { + # Find the pid of the daemon, which we expect is running on the + # background somewhere within our current session. + SERVER_PID=`pgrep -s 0 -x chasquid` + + echo $( cat /proc/$SERVER_PID/status | grep VmHWM | cut -d ':' -f 2- ) +} diff --git a/test/util/loadgen.go b/test/util/loadgen.go new file mode 100644 index 0000000..ca5c0ec --- /dev/null +++ b/test/util/loadgen.go @@ -0,0 +1,195 @@ +// +build ignore + +// SMTP load generator, for testing purposes. +package main + +import ( + "flag" + "net" + "net/http" + "runtime" + "sync" + "time" + + _ "net/http/pprof" + + "golang.org/x/net/trace" + + "blitiri.com.ar/go/chasquid/internal/smtp" + "blitiri.com.ar/go/log" +) + +var ( + addr = flag.String("addr", "", + "server address") + httpAddr = flag.String("http_addr", "localhost:8011", + "monitoring HTTP server listening address") + parallel = flag.Int("parallel", 0, + "how many sending loops to run in parallel") + runFor = flag.Duration("run_for", 0, + "how long to run for (0 = forever)") + wait = flag.Bool("wait", false, + "don't exit after --run_for has lapsed") + noop = flag.Bool("noop", false, + "don't send an email, just connect and run a NOOP") +) + +var ( + host string + exit bool + + globalCount int64 = 0 + globalRuntime time.Duration + globalMu = &sync.Mutex{} +) + +func main() { + var err error + + flag.Parse() + log.Init() + + host, _, err = net.SplitHostPort(*addr) + if err != nil { + log.Fatalf("failed to split --addr=%q: %v", *addr, err) + } + + if *wait { + go http.ListenAndServe(*httpAddr, nil) + log.Infof("monitoring address: http://%v/debug/requests?fam=one&b=11", + *httpAddr) + } + + if *parallel == 0 { + *parallel = runtime.GOMAXPROCS(0) + } + + lt := "full" + if *noop { + lt = "noop" + } + + log.Infof("launching %d %s sending loops in parallel", *parallel, lt) + for i := 0; i < *parallel; i++ { + go serial(i) + } + + var totalCount int64 + var totalRuntime time.Duration + start := time.Now() + for range time.Tick(1 * time.Second) { + globalMu.Lock() + totalCount += globalCount + totalRuntime += globalRuntime + count := globalCount + runtime := globalRuntime + globalCount = 0 + globalRuntime = 0 + globalMu.Unlock() + + if count == 0 { + log.Infof("0 ops") + } else { + log.Infof("%d ops, %v /op", count, + time.Duration(runtime.Nanoseconds()/count).Truncate(time.Microsecond)) + } + + if *runFor > 0 && time.Since(start) > *runFor { + exit = true + break + } + } + + end := time.Now() + window := end.Sub(start) + log.Infof("total: %d ops, %v wall, %v run", + totalCount, + window.Truncate(time.Millisecond), + totalRuntime.Truncate(time.Millisecond)) + + avgLat := time.Duration(totalRuntime.Nanoseconds() / totalCount) + log.Infof("avg: %v /op, %.0f ops/s", + avgLat.Truncate(time.Microsecond), + float64(totalCount)/window.Seconds(), + ) + + if *wait { + for { + time.Sleep(24 * time.Hour) + } + } +} + +func serial(id int) { + var count int64 + start := time.Now() + for { + count += 1 + err := one() + if err != nil { + log.Fatalf("%v", err) + } + + if count == 5 { + globalMu.Lock() + globalCount += count + globalRuntime += time.Since(start) + globalMu.Unlock() + count = 0 + start = time.Now() + + if exit { + return + } + } + } +} + +func one() error { + tr := trace.New("one", *addr) + defer tr.Finish() + + conn, err := net.Dial("tcp", *addr) + if err != nil { + return err + } + defer conn.Close() + + client, err := smtp.NewClient(conn, host) + if err != nil { + return err + } + defer client.Close() + + if *noop { + err = client.Noop() + if err != nil { + return err + } + } else { + err = client.MailAndRcpt("test@test", "null@testserver") + if err != nil { + return err + } + + w, err := client.Data() + if err != nil { + return err + } + _, err = w.Write(body) + if err != nil { + return err + } + err = w.Close() + if err != nil { + return err + } + } + + return nil +} + +var body = []byte(`Subject: Load test + +This is the body of the load test email. +`)