diff --git a/internal/smtpsrv/conn.go b/internal/smtpsrv/conn.go
index 479fb59..d3fe30a 100644
--- a/internal/smtpsrv/conn.go
+++ b/internal/smtpsrv/conn.go
@@ -163,6 +163,10 @@ func (c *Conn) Handle() {
 	defer c.tr.Finish()
 	c.tr.Debugf("Connected, mode: %s", c.mode)
 
+	// Set the first deadline, which covers possibly the TLS handshake and
+	// then our initial greeting.
+	c.conn.SetDeadline(time.Now().Add(c.commandTimeout))
+
 	if tc, ok := c.conn.(*tls.Conn); ok {
 		// For TLS connections, complete the handshake and get the state, so
 		// it can be used when we say hello below.