From 894db04d702a2fddc29e48d724bcb574d0832832 Mon Sep 17 00:00:00 2001 From: kingforaday <38838211+kingforaday@users.noreply.github.com> Date: Sun, 6 May 2018 14:56:38 -0400 Subject: [PATCH 01/19] Opportunistic TLS Support (#98) * STARTTLS Support, disabled by default. * Added documentation --- doc/config.md | 33 +++++++ pkg/config/config.go | 3 + pkg/server/smtp/handler.go | 168 ++++++++++++++++++++---------------- pkg/server/smtp/listener.go | 18 ++++ 4 files changed, 150 insertions(+), 72 deletions(-) diff --git a/doc/config.md b/doc/config.md index f932d01..e003d3a 100644 --- a/doc/config.md +++ b/doc/config.md @@ -21,6 +21,9 @@ variables it supports: INBUCKET_SMTP_STOREDOMAINS Domains to store mail for INBUCKET_SMTP_DISCARDDOMAINS Domains to discard mail for INBUCKET_SMTP_TIMEOUT 300s Idle network timeout + INBUCKET_SMTP_TLSENABLED false Enable STARTTLS option + INBUCKET_SMTP_TLSPRIVKEY cert.key X509 Private Key file for TLS Support + INBUCKET_SMTP_TLSCERT cert.crt X509 Public Certificate file for TLS Support INBUCKET_POP3_ADDR 0.0.0.0:1100 POP3 server IP4 host:port INBUCKET_POP3_DOMAIN inbucket HELLO domain INBUCKET_POP3_TIMEOUT 600s Idle network timeout @@ -202,6 +205,36 @@ to the public internet. - Default: `300s` - Values: Duration ending in `s` for seconds, `m` for minutes +### TLS Support Availability + +`INBUCKET_SMTP_TLSENABLED` + +Enable the STARTTLS option for opportunistic TLS support + +- Default: `false` +- Values: `true` or `false` + +### TLS Private Key File + +`INBUCKET_SMTP_TLSPRIVKEY` + +Specify the x509 Private key file to be used for TLS negotiation. +This option is only valid when INBUCKET_SMTP_TLSENABLED is enabled. + +- Default: `cert.key` +- Values: filename or path to private key +- Example: `server.privkey` + +### TLS Public Certificate File + +`INBUCKET_SMTP_TLSPRIVKEY` + +Specify the x509 Certificate file to be used for TLS negotiation. +This option is only valid when INBUCKET_SMTP_TLSENABLED is enabled. + +- Default: `cert.crt` +- Values: filename or path to the certificate key +- Example: `server.crt` ## POP3 diff --git a/pkg/config/config.go b/pkg/config/config.go index 3071907..9c1228f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -76,6 +76,9 @@ type SMTP struct { StoreDomains []string `desc:"Domains to store mail for"` DiscardDomains []string `desc:"Domains to discard mail for"` Timeout time.Duration `required:"true" default:"300s" desc:"Idle network timeout"` + TLSEnabled bool `default:"false" desc:"Enable STARTTLS option"` + TLSPrivKey string `default:"cert.key" desc:"X509 Private Key file for TLS Support"` + TLSCert string `default:"cert.crt" desc:"X509 Public Certificate file for TLS Support"` Debug bool `ignored:"true"` } diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index ea0a801..cb54e8e 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -3,9 +3,11 @@ package smtp import ( "bufio" "bytes" + "crypto/tls" "fmt" "io" "net" + "net/textproto" "regexp" "strconv" "strings" @@ -58,21 +60,22 @@ func (s State) String() string { } var commands = map[string]bool{ - "HELO": true, - "EHLO": true, - "MAIL": true, - "RCPT": true, - "DATA": true, - "RSET": true, - "SEND": true, - "SOML": true, - "SAML": true, - "VRFY": true, - "EXPN": true, - "HELP": true, - "NOOP": true, - "QUIT": true, - "TURN": true, + "HELO": true, + "EHLO": true, + "MAIL": true, + "RCPT": true, + "DATA": true, + "RSET": true, + "SEND": true, + "SOML": true, + "SAML": true, + "VRFY": true, + "EXPN": true, + "HELP": true, + "NOOP": true, + "QUIT": true, + "TURN": true, + "STARTTLS": true, } // Session holds the state of an SMTP session @@ -89,12 +92,15 @@ type Session struct { recipients []*policy.Recipient // Recipients from RCPT commands. logger zerolog.Logger // Session specific logger. debug bool // Print network traffic to stdout. + tlsState *tls.ConnectionState + text *textproto.Conn } // NewSession creates a new Session for the given connection func NewSession(server *Server, id int, conn net.Conn, logger zerolog.Logger) *Session { reader := bufio.NewReader(conn) host, _, _ := net.SplitHostPort(conn.RemoteAddr().String()) + return &Session{ Server: server, id: id, @@ -105,6 +111,7 @@ func NewSession(server *Server, id int, conn net.Conn, logger zerolog.Logger) *S recipients: make([]*policy.Recipient, 0), logger: logger, debug: server.config.Debug, + text: textproto.NewConn(conn), } } @@ -135,6 +142,7 @@ func (s *Server) startSession(id int, conn net.Conn) { }() ssn := NewSession(s, id, conn, logger) + defer ssn.text.Close() ssn.greet() // This is our command reading loop @@ -232,6 +240,7 @@ func (s *Server) startSession(id int, conn net.Conn) { // GREET state -> waiting for HELO func (s *Session) greetHandler(cmd string, arg string) { + const readyBanner = "Great, let's get this show on the road" switch cmd { case "HELO": domain, err := parseHelloArgument(arg) @@ -240,7 +249,7 @@ func (s *Session) greetHandler(cmd string, arg string) { return } s.remoteDomain = domain - s.send("250 Great, let's get this show on the road") + s.send("250 " + readyBanner) s.enterState(READY) case "EHLO": domain, err := parseHelloArgument(arg) @@ -249,8 +258,12 @@ func (s *Session) greetHandler(cmd string, arg string) { return } s.remoteDomain = domain - s.send("250-Great, let's get this show on the road") + // features before SIZE per RFC + s.send("250-" + readyBanner) s.send("250-8BITMIME") + if s.Server.config.TLSEnabled && s.Server.tlsConfig != nil && s.tlsState == nil { + s.send("250-STARTTLS") + } s.send(fmt.Sprintf("250 SIZE %v", s.config.MaxMessageBytes)) s.enterState(READY) default: @@ -271,7 +284,29 @@ func parseHelloArgument(arg string) (string, error) { // READY state -> waiting for MAIL func (s *Session) readyHandler(cmd string, arg string) { - if cmd == "MAIL" { + if cmd == "STARTTLS" { + if !s.Server.config.TLSEnabled { + // invalid command since unconfigured + s.logger.Debug().Msgf("454 TLS unavailable on the server") + s.send("454 TLS unavailable on the server") + return + } + if s.tlsState != nil { + // tls state previously valid + s.logger.Debug().Msg("454 A TLS session already agreed upon.") + s.send("454 A TLS session already agreed upon.") + return + } + s.logger.Debug().Msg("Initiating TLS context.") + s.send("220 STARTTLS") + // start tls connection handshake + tlsConn := tls.Server(s.conn, s.Server.tlsConfig) + s.conn = tlsConn + s.text = textproto.NewConn(s.conn) + s.tlsState = new(tls.ConnectionState) + *s.tlsState = tlsConn.ConnectionState() + s.enterState(GREET) + } else if cmd == "MAIL" { // Capture group 1: from address. 2: optional params. m := fromRegex.FindStringSubmatch(arg) if m == nil { @@ -367,57 +402,43 @@ func (s *Session) mailHandler(cmd string, arg string) { // DATA func (s *Session) dataHandler() { s.send("354 Start mail input; end with .") - msgBuf := &bytes.Buffer{} - for { - lineBuf, err := s.readByteLine() - if err != nil { - if netErr, ok := err.(net.Error); ok { - if netErr.Timeout() { - s.send("221 Idle timeout, bye bye") - } + msgBuf, err := s.readByteLine() + if err != nil { + if netErr, ok := err.(net.Error); ok { + if netErr.Timeout() { + s.send("221 Idle timeout, bye bye") } - s.logger.Warn().Msgf("Error: %v while reading", err) - s.enterState(QUIT) - return - } - if bytes.Equal(lineBuf, []byte(".\r\n")) || bytes.Equal(lineBuf, []byte(".\n")) { - // Mail data complete. - tstamp := time.Now().Format(timeStampFormat) - for _, recip := range s.recipients { - if recip.ShouldStore() { - // Generate Received header. - prefix := fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", - s.remoteDomain, s.remoteHost, s.config.Domain, recip.Address.Address, - tstamp) - // Deliver message. - _, err := s.manager.Deliver( - recip, s.from, s.recipients, prefix, msgBuf.Bytes()) - if err != nil { - s.logger.Error().Msgf("delivery for %v: %v", recip.LocalPart, err) - s.send(fmt.Sprintf("451 Failed to store message for %v", recip.LocalPart)) - s.reset() - return - } - } - expReceivedTotal.Add(1) - } - s.send("250 Mail accepted for delivery") - s.logger.Info().Msgf("Message size %v bytes", msgBuf.Len()) - s.reset() - return - } - // RFC: remove leading periods from DATA. - if len(lineBuf) > 0 && lineBuf[0] == '.' { - lineBuf = lineBuf[1:] - } - msgBuf.Write(lineBuf) - if msgBuf.Len() > s.config.MaxMessageBytes { - s.send("552 Maximum message size exceeded") - s.logger.Warn().Msgf("Max message size exceeded while in DATA") - s.reset() - return } + s.logger.Warn().Msgf("Error: %v while reading", err) + s.enterState(QUIT) + return } + mailData := bytes.NewBuffer(msgBuf) + + // Mail data complete. + tstamp := time.Now().Format(timeStampFormat) + for _, recip := range s.recipients { + if recip.ShouldStore() { + // Generate Received header. + prefix := fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", + s.remoteDomain, s.remoteHost, s.config.Domain, recip.Address.Address, + tstamp) + // Deliver message. + _, err := s.manager.Deliver( + recip, s.from, s.recipients, prefix, mailData.Bytes()) + if err != nil { + s.logger.Error().Msgf("delivery for %v: %v", recip.LocalPart, err) + s.send(fmt.Sprintf("451 Failed to store message for %v", recip.LocalPart)) + s.reset() + return + } + } + expReceivedTotal.Add(1) + } + s.send("250 Mail accepted for delivery") + s.logger.Info().Msgf("Message size %v bytes", mailData.Len()) + s.reset() + return } func (s *Session) enterState(state State) { @@ -440,7 +461,7 @@ func (s *Session) send(msg string) { s.sendError = err return } - if _, err := fmt.Fprint(s.conn, msg+"\r\n"); err != nil { + if err := s.text.PrintfLine("%s", msg); err != nil { s.sendError = err s.logger.Warn().Msgf("Failed to send: %q", msg) return @@ -455,9 +476,12 @@ func (s *Session) readByteLine() ([]byte, error) { if err := s.conn.SetReadDeadline(s.nextDeadline()); err != nil { return nil, err } - b, err := s.reader.ReadBytes('\n') - if err == nil && s.debug { - fmt.Printf("%04d %s\n", s.id, bytes.TrimRight(b, "\r\n")) + b, err := s.text.ReadDotBytes() + if err != nil { + return nil, err + } + if s.debug { + fmt.Printf("%04d Received %d bytes\n", s.id, len(b)) } return b, err } @@ -467,7 +491,7 @@ func (s *Session) readLine() (line string, err error) { if err = s.conn.SetReadDeadline(s.nextDeadline()); err != nil { return "", err } - line, err = s.reader.ReadString('\n') + line, err = s.text.ReadLine() if err != nil { return "", err } @@ -486,7 +510,7 @@ func (s *Session) parseCmd(line string) (cmd string, arg string, ok bool) { case l < 4: s.logger.Warn().Msgf("Command too short: %q", line) return "", "", false - case l == 4: + case l == 4 || l == 8: return strings.ToUpper(line), "", true case l == 5: // Too long to be only command, too short to have args diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index 9161d75..a4d3a3f 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -3,6 +3,7 @@ package smtp import ( "container/list" "context" + "crypto/tls" "expvar" "net" "sync" @@ -63,6 +64,7 @@ type Server struct { manager message.Manager // Used to deliver messages. listener net.Listener // Incoming network connections. wg *sync.WaitGroup // Waitgroup tracks individual sessions. + tlsConfig *tls.Config } // NewServer creates a new Server instance with the specificed config. @@ -72,12 +74,28 @@ func NewServer( manager message.Manager, apolicy *policy.Addressing, ) *Server { + slog := log.With().Str("module", "smtp").Str("phase", "tls").Logger() + tlsConfig := &tls.Config{} + if smtpConfig.TLSEnabled { + var err error + tlsConfig.Certificates = make([]tls.Certificate, 1) + tlsConfig.Certificates[0], err = tls.LoadX509KeyPair(smtpConfig.TLSCert, smtpConfig.TLSPrivKey) + if err != nil { + slog.Error().Msgf("Failed loading X509 KeyPair: %v", err) + slog.Error().Msg("Disabling STARTTLS support") + smtpConfig.TLSEnabled = false + } else { + slog.Debug().Msg("STARTTLS feature available") + } + } + return &Server{ config: smtpConfig, globalShutdown: globalShutdown, manager: manager, addrPolicy: apolicy, wg: new(sync.WaitGroup), + tlsConfig: tlsConfig, } } From fdcb29a52b1c339afeb9b841d79cabd12cfb31dd Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 6 May 2018 12:09:55 -0700 Subject: [PATCH 02/19] smtp: rename readByteLine to readDataBlock for #98. Update change log. --- CHANGELOG.md | 8 ++++++++ pkg/server/smtp/handler.go | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb3ddca..8e95406 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,20 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). + +## [Unreleased] + +### Added +- SMTP TLS support (thanks kingforaday.) + + ## [v2.0.0] - 2018-05-05 ### Changed - Corrected docs for INBUCKET_STORAGE_PARAMS (thanks evilmrburns.) - Disabled color log output on Windows, doesn't work there. + ## [v2.0.0-rc1] - 2018-04-07 ### Added diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index cb54e8e..ad7c87b 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -402,7 +402,7 @@ func (s *Session) mailHandler(cmd string, arg string) { // DATA func (s *Session) dataHandler() { s.send("354 Start mail input; end with .") - msgBuf, err := s.readByteLine() + msgBuf, err := s.readDataBlock() if err != nil { if netErr, ok := err.(net.Error); ok { if netErr.Timeout() { @@ -471,8 +471,8 @@ func (s *Session) send(msg string) { } } -// readByteLine reads a line of input, returns byte slice. -func (s *Session) readByteLine() ([]byte, error) { +// readDataBlock reads message DATA until `.` using the textproto pkg. +func (s *Session) readDataBlock() ([]byte, error) { if err := s.conn.SetReadDeadline(s.nextDeadline()); err != nil { return nil, err } @@ -486,7 +486,7 @@ func (s *Session) readByteLine() ([]byte, error) { return b, err } -// Reads a line of input +// readLine reads a line of input respecting deadlines. func (s *Session) readLine() (line string, err error) { if err = s.conn.SetReadDeadline(s.nextDeadline()); err != nil { return "", err From 00dad88bde48301460742f2944a0c531aa21b5b5 Mon Sep 17 00:00:00 2001 From: kingforaday <38838211+kingforaday@users.noreply.github.com> Date: Sun, 20 May 2018 12:51:40 -0400 Subject: [PATCH 03/19] Fixing an erroneous connection close introduced in #98. (#101) --- pkg/server/smtp/handler.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index ad7c87b..f513a62 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -142,7 +142,6 @@ func (s *Server) startSession(id int, conn net.Conn) { }() ssn := NewSession(s, id, conn, logger) - defer ssn.text.Close() ssn.greet() // This is our command reading loop From 0d7c94c5315a96361f9b397a6e5a3821df1349ea Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 20 Oct 2018 11:13:39 -0700 Subject: [PATCH 04/19] smtp: add missing log message param --- pkg/server/smtp/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index f513a62..fcae44e 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -536,7 +536,7 @@ func (s *Session) parseArgs(arg string) (args map[string]string, ok bool) { re := regexp.MustCompile(` (\w+)=(\w+)`) pm := re.FindAllStringSubmatch(arg, -1) if pm == nil { - s.logger.Warn().Msgf("Failed to parse arg string: %q") + s.logger.Warn().Msgf("Failed to parse arg string: %q", arg) return nil, false } for _, m := range pm { From bf12925fd1916c6a75a858597205efdb5c161c19 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 20 Oct 2018 11:18:48 -0700 Subject: [PATCH 05/19] travis: golint & golang updates --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7d697ca..ce8d253 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,11 +10,12 @@ env: - DEPLOY_WITH_MAJOR="1.10" before_script: - - go get github.com/golang/lint/golint + - go get golang.org/x/lint/golint - make deps go: - - "1.10.1" + - "1.10.x" + - "1.11.x" deploy: provider: script From dc007da82e42a1891390cb042ea64e8d5e78cbac Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 20 Oct 2018 11:50:25 -0700 Subject: [PATCH 06/19] build: Use go modules for #121 - travis: Bump release trigger env to 1.11 --- .travis.yml | 4 +++- CHANGELOG.md | 1 + go.mod | 18 ++++++++++++++++++ go.sum | 40 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 go.mod create mode 100644 go.sum diff --git a/.travis.yml b/.travis.yml index ce8d253..1844167 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,9 @@ addons: - rpm env: - - DEPLOY_WITH_MAJOR="1.10" + global: + - GO111MODULE=on + - DEPLOY_WITH_MAJOR="1.11" before_script: - go get golang.org/x/lint/golint diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e95406..2605f34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] ### Added +- Use Go 1.11 modules for reproducible builds. - SMTP TLS support (thanks kingforaday.) diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..4291717 --- /dev/null +++ b/go.mod @@ -0,0 +1,18 @@ +module github.com/jhillyerd/inbucket + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/google/subcommands v0.0.0-20181012225330-46f0354f6315 + github.com/gorilla/css v1.0.0 + github.com/gorilla/mux v1.6.2 + github.com/gorilla/securecookie v1.1.1 + github.com/gorilla/sessions v1.1.3 + github.com/gorilla/websocket v1.4.0 + github.com/jhillyerd/enmime v0.2.1 + github.com/kelseyhightower/envconfig v1.3.0 + github.com/microcosm-cc/bluemonday v1.0.1 + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/zerolog v1.9.1 + github.com/stretchr/testify v1.2.2 + golang.org/x/net v0.0.0-20181017193950-04a2e542c03f +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..7b3fe17 --- /dev/null +++ b/go.sum @@ -0,0 +1,40 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/subcommands v0.0.0-20181012225330-46f0354f6315 h1:WW91Hq2v0qDzoPME+TPD4En72+d2Ue3ZMKPYfwR9yBU= +github.com/google/subcommands v0.0.0-20181012225330-46f0354f6315/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= +github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.1.3 h1:uXoZdcdA5XdXF3QzuSlheVRUvjl+1rKY7zBXL68L9RU= +github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= +github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ= +github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= +github.com/jhillyerd/enmime v0.2.1 h1:YodBfMH3jmrZn68Gg4ZoZH1ECDsdh8BLW9+DjoFce6o= +github.com/jhillyerd/enmime v0.2.1/go.mod h1:0gWUCFBL87cvx6/MSSGNBHJ6r+fMArqltDFwHxC10P4= +github.com/kelseyhightower/envconfig v1.3.0 h1:IvRS4f2VcIQy6j4ORGIf9145T/AsUB+oY8LyvN8BXNM= +github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= +github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/microcosm-cc/bluemonday v1.0.1 h1:SIYunPjnlXcW+gVfvm0IlSeR5U3WZUOLfVmqg85Go44= +github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= +github.com/olekukonko/tablewriter v0.0.0-20180912035003-be2c049b30cc h1:rQ1O4ZLYR2xXHXgBCCfIIGnuZ0lidMQw2S5n1oOv+Wg= +github.com/olekukonko/tablewriter v0.0.0-20180912035003-be2c049b30cc/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/zerolog v1.9.1 h1:AjV/SFRF0+gEa6rSjkh0Eji/DnkrJKVpPho6SW5g4mU= +github.com/rs/zerolog v1.9.1/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= +github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/net v0.0.0-20181017193950-04a2e542c03f h1:4pRM7zYwpBjCnfA1jRmhItLxYJkaEnsmuAcRtA347DA= +golang.org/x/net v0.0.0-20181017193950-04a2e542c03f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From f9adced65e1f93305b2493424651c6d7efe305f8 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 20 Oct 2018 12:30:04 -0700 Subject: [PATCH 07/19] docker: Update build to use Go 1.11+modules for #121 - Fix outdated params in docker-run.sh --- CHANGELOG.md | 3 +++ Dockerfile | 9 ++++----- etc/docker/docker-run.sh | 14 +++++++------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2605f34..24b7a16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ This project adheres to [Semantic Versioning](http://semver.org/). - Use Go 1.11 modules for reproducible builds. - SMTP TLS support (thanks kingforaday.) +### Changed +- Docker build now uses Go 1.11 and Alpine 3.8 + ## [v2.0.0] - 2018-05-05 diff --git a/Dockerfile b/Dockerfile index 9a1e0b9..0bc28fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ # Docker build file for Inbucket: https://www.inbucket.org/ # Build -FROM golang:1.10-alpine as builder +FROM golang:1.11-alpine3.8 as builder RUN apk add --no-cache --virtual .build-deps git make -WORKDIR /go/src/github.com/jhillyerd/inbucket +WORKDIR /build COPY . . ENV CGO_ENABLED 0 RUN make clean deps @@ -12,11 +12,10 @@ RUN go build -o inbucket \ -v ./cmd/inbucket # Run in minimal image -FROM alpine:3.7 -ENV SRC /go/src/github.com/jhillyerd/inbucket +FROM alpine:3.8 WORKDIR /opt/inbucket RUN mkdir bin defaults ui -COPY --from=builder $SRC/inbucket bin +COPY --from=builder /build/inbucket bin COPY etc/docker/defaults/greeting.html defaults COPY ui ui COPY etc/docker/defaults/start-inbucket.sh / diff --git a/etc/docker/docker-run.sh b/etc/docker/docker-run.sh index 442f2fe..26c020c 100755 --- a/etc/docker/docker-run.sh +++ b/etc/docker/docker-run.sh @@ -12,9 +12,9 @@ PORT_POP3=1100 # Volumes exposed on host: VOL_CONFIG="/tmp/inbucket/config" -VOL_DATA="/tmp/inbucket/data" +VOL_DATA="/tmp/inbucket/storage" -set -eo pipefail +set -e main() { local run_opts="" @@ -39,11 +39,11 @@ main() { done docker run $run_opts \ - -p $PORT_HTTP:10080 \ - -p $PORT_SMTP:10025 \ - -p $PORT_POP3:10110 \ - -v "$VOL_CONFIG:/con/configuration" \ - -v "$VOL_DATA:/con/data" \ + -p $PORT_HTTP:9000 \ + -p $PORT_SMTP:2500 \ + -p $PORT_POP3:1100 \ + -v "$VOL_CONFIG:/config" \ + -v "$VOL_DATA:/storage" \ "$IMAGE" } From 98745b3bb9652c3e7cddee0f929fbac19702bf18 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 20 Oct 2018 16:16:09 -0700 Subject: [PATCH 08/19] web: Optionally mount /debug/pprof for #120 - web: eliminate use of http.DefaultServeMux --- CHANGELOG.md | 1 + doc/config.md | 15 +++++++++++++++ pkg/config/config.go | 1 + pkg/server/web/server.go | 14 ++++++++++++-- 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24b7a16..369f279 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Added - Use Go 1.11 modules for reproducible builds. - SMTP TLS support (thanks kingforaday.) +- `INBUCKET_WEB_PPROF` configuration option for performance profiling. ### Changed - Docker build now uses Go 1.11 and Alpine 3.8 diff --git a/doc/config.md b/doc/config.md index e003d3a..c880d4e 100644 --- a/doc/config.md +++ b/doc/config.md @@ -35,6 +35,7 @@ variables it supports: INBUCKET_WEB_COOKIEAUTHKEY Session cipher key (text) INBUCKET_WEB_MONITORVISIBLE true Show monitor tab in UI? INBUCKET_WEB_MONITORHISTORY 30 Monitor remembered messages + INBUCKET_WEB_PPROF false Expose profiling tools on /debug/pprof INBUCKET_STORAGE_TYPE memory Storage impl: file or memory INBUCKET_STORAGE_PARAMS Storage impl parameters, see docs. INBUCKET_STORAGE_RETENTIONPERIOD 24h Duration to retain messages @@ -377,6 +378,20 @@ them. - Default: `30` - Values: Integer greater than or equal to 0 +### Performance Profiling & Debug Tools + +`INBUCKET_WEB_PPROF` + +If true, Go's pprof package will be installed to the `/debug/pprof` URI. This +exposes detailed memory and CPU performance data for debugging Inbucket. If you +enable this option, please make sure it is not exposed to the public internet, +as its use can significantly impact performance. + +For example usage, see https://golang.org/pkg/net/http/pprof/ + +- Default: `false` +- Values: `true` or `false` + ## Storage diff --git a/pkg/config/config.go b/pkg/config/config.go index 9c1228f..b1bf39a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -100,6 +100,7 @@ type Web struct { CookieAuthKey string `desc:"Session cipher key (text)"` MonitorVisible bool `required:"true" default:"true" desc:"Show monitor tab in UI?"` MonitorHistory int `required:"true" default:"30" desc:"Monitor remembered messages"` + PProf bool `required:"true" default:"false" desc:"Expose profiling tools on /debug/pprof"` } // Storage contains the mail store configuration. diff --git a/pkg/server/web/server.go b/pkg/server/web/server.go index a60735a..bc7bc5e 100644 --- a/pkg/server/web/server.go +++ b/pkg/server/web/server.go @@ -6,6 +6,7 @@ import ( "expvar" "net" "net/http" + "net/http/pprof" "path/filepath" "time" @@ -70,7 +71,16 @@ func Initialize( Msg("Web UI content mapped") Router.PathPrefix("/public/").Handler(http.StripPrefix("/public/", http.FileServer(http.Dir(staticPath)))) - http.Handle("/", Router) + Router.Handle("/debug/vars", expvar.Handler()) + if conf.Web.PProf { + Router.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + Router.HandleFunc("/debug/pprof/profile", pprof.Profile) + Router.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + Router.HandleFunc("/debug/pprof/trace", pprof.Trace) + Router.PathPrefix("/debug/pprof/").HandlerFunc(pprof.Index) + log.Warn().Str("module", "web").Str("phase", "startup"). + Msg("Go pprof tools installed to /debug/pprof") + } // Session cookie setup if conf.Web.CookieAuthKey == "" { @@ -88,7 +98,7 @@ func Initialize( func Start(ctx context.Context) { server = &http.Server{ Addr: rootConfig.Web.Addr, - Handler: nil, + Handler: Router, ReadTimeout: 60 * time.Second, WriteTimeout: 60 * time.Second, } From f68f07d896a677b4709e1c4be9b07364ff7c385b Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 20 Oct 2018 20:33:27 -0700 Subject: [PATCH 09/19] file: pool index readers to reduce allocs for #122 --- pkg/storage/file/fstore.go | 33 ++++++++++++++++++++++++++++----- pkg/storage/file/mbox.go | 4 +++- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index a52bc36..2eb51c9 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "os" "path/filepath" + "sync" "time" "github.com/jhillyerd/inbucket/pkg/config" @@ -40,10 +41,11 @@ func countGenerator(c chan int) { // Store implements DataStore aand is the root of the mail storage // hiearchy. It provides access to Mailbox objects type Store struct { - hashLock storage.HashLock - path string - mailPath string - messageCap int + hashLock storage.HashLock + path string + mailPath string + messageCap int + bufReaderPool sync.Pool } // New creates a new DataStore object using the specified path @@ -60,7 +62,16 @@ func New(cfg config.Storage) (storage.Store, error) { Msg("Error creating dir") } } - return &Store{path: path, mailPath: mailPath, messageCap: cfg.MailboxMsgCap}, nil + return &Store{ + path: path, + mailPath: mailPath, + messageCap: cfg.MailboxMsgCap, + bufReaderPool: sync.Pool{ + New: func() interface{} { + return bufio.NewReader(nil) + }, + }, + }, nil } // AddMessage adds a message to the specified mailbox. @@ -253,6 +264,18 @@ func (fs *Store) mboxFromHash(hash string) *mbox { } } +// getPooledReader pulls a buffered reader from the fs.bufReaderPool +func (fs *Store) getPooledReader(r io.Reader) *bufio.Reader { + br := fs.bufReaderPool.Get().(*bufio.Reader) + br.Reset(r) + return br +} + +// putPooledReader returns a buffered reader to the fs.bufReaderPool +func (fs *Store) putPooledReader(br *bufio.Reader) { + fs.bufReaderPool.Put(br) +} + // generatePrefix converts a Time object into the ISO style format we use // as a prefix for message files. Note: It is used directly by unit // tests. diff --git a/pkg/storage/file/mbox.go b/pkg/storage/file/mbox.go index 8c85145..55f1c9f 100644 --- a/pkg/storage/file/mbox.go +++ b/pkg/storage/file/mbox.go @@ -120,7 +120,9 @@ func (mb *mbox) readIndex() error { } }() // Decode gob data - dec := gob.NewDecoder(bufio.NewReader(file)) + br := mb.store.getPooledReader(file) + defer mb.store.putPooledReader(br) + dec := gob.NewDecoder(br) name := "" if err = dec.Decode(&name); err != nil { return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) From 0640f9fa08febf79068afe52f02889a15821a3cf Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 21 Oct 2018 09:25:32 -0700 Subject: [PATCH 10/19] file: Use os.Readdirnames to eliminate Lstat calls for #122 - This a speed/syscall optimization, not memory. --- pkg/storage/file/fstore.go | 64 +++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index 2eb51c9..5d3dcc7 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -4,7 +4,6 @@ import ( "bufio" "fmt" "io" - "io/ioutil" "os" "path/filepath" "sync" @@ -190,41 +189,33 @@ func (fs *Store) PurgeMessages(mailbox string) error { // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it // continues to return true. func (fs *Store) VisitMailboxes(f func([]storage.Message) (cont bool)) error { - infos1, err := ioutil.ReadDir(fs.mailPath) + names1, err := readDirNames(fs.mailPath) if err != nil { return err } // Loop over level 1 directories - for _, inf1 := range infos1 { - if inf1.IsDir() { - l1 := inf1.Name() - infos2, err := ioutil.ReadDir(filepath.Join(fs.mailPath, l1)) + for _, name1 := range names1 { + names2, err := readDirNames(fs.mailPath, name1) + if err != nil { + return err + } + // Loop over level 2 directories + for _, name2 := range names2 { + names3, err := readDirNames(fs.mailPath, name1, name2) if err != nil { return err } - // Loop over level 2 directories - for _, inf2 := range infos2 { - if inf2.IsDir() { - l2 := inf2.Name() - infos3, err := ioutil.ReadDir(filepath.Join(fs.mailPath, l1, l2)) - if err != nil { - return err - } - // Loop over mailboxes - for _, inf3 := range infos3 { - if inf3.IsDir() { - mb := fs.mboxFromHash(inf3.Name()) - mb.RLock() - msgs, err := mb.getMessages() - mb.RUnlock() - if err != nil { - return err - } - if !f(msgs) { - return nil - } - } - } + // Loop over mailboxes + for _, name3 := range names3 { + mb := fs.mboxFromHash(name3) + mb.RLock() + msgs, err := mb.getMessages() + mb.RUnlock() + if err != nil { + return err + } + if !f(msgs) { + return nil } } } @@ -264,14 +255,14 @@ func (fs *Store) mboxFromHash(hash string) *mbox { } } -// getPooledReader pulls a buffered reader from the fs.bufReaderPool +// getPooledReader pulls a buffered reader from the fs.bufReaderPool. func (fs *Store) getPooledReader(r io.Reader) *bufio.Reader { br := fs.bufReaderPool.Get().(*bufio.Reader) br.Reset(r) return br } -// putPooledReader returns a buffered reader to the fs.bufReaderPool +// putPooledReader returns a buffered reader to the fs.bufReaderPool. func (fs *Store) putPooledReader(br *bufio.Reader) { fs.bufReaderPool.Put(br) } @@ -284,7 +275,16 @@ func generatePrefix(date time.Time) string { } // generateId adds a 4-digit unique number onto the end of the string -// returned by generatePrefix() +// returned by generatePrefix(). func generateID(date time.Time) string { return generatePrefix(date) + "-" + fmt.Sprintf("%04d", <-countChannel) } + +// readDirNames returns a slice of filenames in the specified directory or an error. +func readDirNames(elem ...string) ([]string, error) { + f, err := os.Open(filepath.Join(elem...)) + if err != nil { + return nil, err + } + return f.Readdirnames(0) +} From 1a7e47b60a2d692ba2cb2f7d9f78af122b1d3651 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 2 Jun 2018 15:07:55 -0700 Subject: [PATCH 11/19] rest: Make tests easier to read, less logic. --- pkg/rest/apiv1_controller_test.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index 885ef3c..0c52673 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -15,8 +15,6 @@ import ( ) const ( - baseURL = "http://localhost/api/v1" - // JSON map keys mailboxKey = "mailbox" idKey = "id" @@ -37,7 +35,7 @@ func TestRestMailboxList(t *testing.T) { logbuf := setupWebServer(mm) // Test invalid mailbox name - w, err := testRestGet(baseURL + "/mailbox/foo%20bar") + w, err := testRestGet("http://localhost/api/v1/mailbox/foo%20bar") expectCode := 500 if err != nil { t.Fatal(err) @@ -47,7 +45,7 @@ func TestRestMailboxList(t *testing.T) { } // Test empty mailbox - w, err = testRestGet(baseURL + "/mailbox/empty") + w, err = testRestGet("http://localhost/api/v1/mailbox/empty") expectCode = 200 if err != nil { t.Fatal(err) @@ -57,7 +55,7 @@ func TestRestMailboxList(t *testing.T) { } // Test Mailbox error - w, err = testRestGet(baseURL + "/mailbox/messageserr") + w, err = testRestGet("http://localhost/api/v1/mailbox/messageserr") expectCode = 500 if err != nil { t.Fatal(err) @@ -89,7 +87,7 @@ func TestRestMailboxList(t *testing.T) { mm.AddMessage("good", &message.Message{Metadata: meta2}) // Check return code - w, err = testRestGet(baseURL + "/mailbox/good") + w, err = testRestGet("http://localhost/api/v1/mailbox/good") expectCode = 200 if err != nil { t.Fatal(err) @@ -139,7 +137,7 @@ func TestRestMessage(t *testing.T) { logbuf := setupWebServer(mm) // Test invalid mailbox name - w, err := testRestGet(baseURL + "/mailbox/foo%20bar/0001") + w, err := testRestGet("http://localhost/api/v1/mailbox/foo%20bar/0001") expectCode := 500 if err != nil { t.Fatal(err) @@ -149,7 +147,7 @@ func TestRestMessage(t *testing.T) { } // Test requesting a message that does not exist - w, err = testRestGet(baseURL + "/mailbox/empty/0001") + w, err = testRestGet("http://localhost/api/v1/mailbox/empty/0001") expectCode = 404 if err != nil { t.Fatal(err) @@ -159,7 +157,7 @@ func TestRestMessage(t *testing.T) { } // Test GetMessage error - w, err = testRestGet(baseURL + "/mailbox/messageerr/0001") + w, err = testRestGet("http://localhost/api/v1/mailbox/messageerr/0001") expectCode = 500 if err != nil { t.Fatal(err) @@ -201,7 +199,7 @@ func TestRestMessage(t *testing.T) { mm.AddMessage("good", msg1) // Check return code - w, err = testRestGet(baseURL + "/mailbox/good/0001") + w, err = testRestGet("http://localhost/api/v1/mailbox/good/0001") expectCode = 200 if err != nil { t.Fatal(err) @@ -264,7 +262,7 @@ func TestRestMarkSeen(t *testing.T) { mm.AddMessage("good", &message.Message{Metadata: meta1}) mm.AddMessage("good", &message.Message{Metadata: meta2}) // Mark one read. - w, err := testRestPatch(baseURL+"/mailbox/good/0002", `{"seen":true}`) + w, err := testRestPatch("http://localhost/api/v1/mailbox/good/0002", `{"seen":true}`) expectCode := 200 if err != nil { t.Fatal(err) @@ -273,7 +271,7 @@ func TestRestMarkSeen(t *testing.T) { t.Fatalf("Expected code %v, got %v", expectCode, w.Code) } // Get mailbox. - w, err = testRestGet(baseURL + "/mailbox/good") + w, err = testRestGet("http://localhost/api/v1/mailbox/good") expectCode = 200 if err != nil { t.Fatal(err) From 82e6a9fe5d3a7395e41c6d6337b234b0a982631f Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Mon, 22 Oct 2018 10:48:08 -0700 Subject: [PATCH 12/19] rest: Use a subrouter for /api/ paths --- cmd/inbucket/main.go | 2 +- pkg/rest/routes.go | 16 ++++++++-------- pkg/rest/testutils_test.go | 3 +-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index 88df1ab..ba37c09 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -125,7 +125,7 @@ func main() { retentionScanner.Start() // Start HTTP server. web.Initialize(conf, shutdownChan, mmanager, msgHub) - rest.SetupRoutes(web.Router) + rest.SetupRoutes(web.Router.PathPrefix("/api/").Subrouter()) webui.SetupRoutes(web.Router) go web.Start(rootCtx) // Start POP3 server. diff --git a/pkg/rest/routes.go b/pkg/rest/routes.go index d622f3d..9c4305c 100644 --- a/pkg/rest/routes.go +++ b/pkg/rest/routes.go @@ -6,20 +6,20 @@ import "github.com/jhillyerd/inbucket/pkg/server/web" // SetupRoutes populates the routes for the REST interface func SetupRoutes(r *mux.Router) { // API v1 - r.Path("/api/v1/mailbox/{name}").Handler( + r.Path("/v1/mailbox/{name}").Handler( web.Handler(MailboxListV1)).Name("MailboxListV1").Methods("GET") - r.Path("/api/v1/mailbox/{name}").Handler( + r.Path("/v1/mailbox/{name}").Handler( web.Handler(MailboxPurgeV1)).Name("MailboxPurgeV1").Methods("DELETE") - r.Path("/api/v1/mailbox/{name}/{id}").Handler( + r.Path("/v1/mailbox/{name}/{id}").Handler( web.Handler(MailboxShowV1)).Name("MailboxShowV1").Methods("GET") - r.Path("/api/v1/mailbox/{name}/{id}").Handler( + r.Path("/v1/mailbox/{name}/{id}").Handler( web.Handler(MailboxMarkSeenV1)).Name("MailboxMarkSeenV1").Methods("PATCH") - r.Path("/api/v1/mailbox/{name}/{id}").Handler( + r.Path("/v1/mailbox/{name}/{id}").Handler( web.Handler(MailboxDeleteV1)).Name("MailboxDeleteV1").Methods("DELETE") - r.Path("/api/v1/mailbox/{name}/{id}/source").Handler( + r.Path("/v1/mailbox/{name}/{id}/source").Handler( web.Handler(MailboxSourceV1)).Name("MailboxSourceV1").Methods("GET") - r.Path("/api/v1/monitor/messages").Handler( + r.Path("/v1/monitor/messages").Handler( web.Handler(MonitorAllMessagesV1)).Name("MonitorAllMessagesV1").Methods("GET") - r.Path("/api/v1/monitor/messages/{name}").Handler( + r.Path("/v1/monitor/messages/{name}").Handler( web.Handler(MonitorMailboxMessagesV1)).Name("MonitorMailboxMessagesV1").Methods("GET") } diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index 67b7075..520532c 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -43,7 +43,6 @@ func setupWebServer(mm message.Manager) *bytes.Buffer { log.SetOutput(buf) // Have to reset default mux to prevent duplicate routes - http.DefaultServeMux = http.NewServeMux() cfg := &config.Root{ Web: config.Web{ UIDir: "../ui", @@ -51,7 +50,7 @@ func setupWebServer(mm message.Manager) *bytes.Buffer { } shutdownChan := make(chan bool) web.Initialize(cfg, shutdownChan, mm, &msghub.Hub{}) - SetupRoutes(web.Router) + SetupRoutes(web.Router.PathPrefix("/api/").Subrouter()) return buf } From 2f67a6922aa37bedf22803f2020b6b234a83ce28 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Mon, 22 Oct 2018 12:26:36 -0700 Subject: [PATCH 13/19] ui: Update default greeting.html, closes #106. --- etc/docker/defaults/greeting.html | 2 +- ui/greeting.html | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/etc/docker/defaults/greeting.html b/etc/docker/defaults/greeting.html index 82f3864..84ab514 100644 --- a/etc/docker/defaults/greeting.html +++ b/etc/docker/defaults/greeting.html @@ -1,7 +1,7 @@

Inbucket is an email testing service; it will accept email for any email address and make it available to view without a password.

-

To view email for a particular address, enter the username portion +

To view messages for a particular address, enter the username portion of the address into the box on the upper right and click View.

This instance of Inbucket is running inside of a Inbucket is an email testing service; it will accept email for any email address and make it available to view without a password.

-

To view email for a particular address, enter the username portion +

To view messages for a particular address, enter the username portion of the address into the box on the upper right and click View.

-

This message can be customized by editing greeting.html. Change the -configuration option greeting.file if you'd like to move it -outside of the Inbucket installation directory.

+

This message can be customized by editing ui/greeting.html. +Set the INBUCKET_WEB_GREETINGFILE environment variable if you'd +like to move the file outside of the Inbucket installation directory.

From 8a3d2ff6a23dd28ad5679439cc8f2fb80d7c92f5 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Mon, 22 Oct 2018 15:43:17 -0700 Subject: [PATCH 14/19] storage: Add test for id='latest', implment in mem store. --- pkg/storage/mem/store.go | 11 +++++++++++ pkg/test/storage_suite.go | 26 +++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/pkg/storage/mem/store.go b/pkg/storage/mem/store.go index 47ef265..e8096db 100644 --- a/pkg/storage/mem/store.go +++ b/pkg/storage/mem/store.go @@ -92,6 +92,17 @@ func (s *Store) AddMessage(message storage.Message) (id string, err error) { // GetMessage gets a mesage. func (s *Store) GetMessage(mailbox, id string) (m storage.Message, err error) { + if id == "latest" { + ms, err := s.GetMessages(mailbox) + if err != nil { + return nil, err + } + count := len(ms) + if count == 0 { + return nil, nil + } + return ms[count-1], nil + } s.withMailbox(mailbox, false, func(mb *mbox) { var ok bool m, ok = mb.messages[id] diff --git a/pkg/test/storage_suite.go b/pkg/test/storage_suite.go index 26a82a4..7324b76 100644 --- a/pkg/test/storage_suite.go +++ b/pkg/test/storage_suite.go @@ -27,6 +27,7 @@ func StoreSuite(t *testing.T, factory StoreFactory) { {"metadata", testMetadata, config.Storage{}}, {"content", testContent, config.Storage{}}, {"delivery order", testDeliveryOrder, config.Storage{}}, + {"latest", testLatest, config.Storage{}}, {"naming", testNaming, config.Storage{}}, {"size", testSize, config.Storage{}}, {"seen", testSeen, config.Storage{}}, @@ -192,6 +193,29 @@ func testDeliveryOrder(t *testing.T, store storage.Store) { } } +// testLatest delivers several messages to the same mailbox, and confirms the id `latest` returns +// the last message sent. +func testLatest(t *testing.T, store storage.Store) { + mailbox := "fred" + subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} + for _, subj := range subjects { + DeliverToStore(t, store, mailbox, subj, time.Now()) + } + // Confirm latest. + latest, err := store.GetMessage(mailbox, "latest") + if err != nil { + t.Fatal(err) + } + if latest == nil { + t.Fatalf("Got nil message, wanted most recent message for %v.", mailbox) + } + got := latest.Subject() + want := "echo" + if got != want { + t.Errorf("Got subject %q, want %q", got, want) + } +} + // testNaming ensures the store does not enforce local part mailbox naming. func testNaming(t *testing.T, store storage.Store) { DeliverToStore(t, store, "fred@fish.net", "disk #27", time.Now()) @@ -199,7 +223,7 @@ func testNaming(t *testing.T, store storage.Store) { GetAndCountMessages(t, store, "fred@fish.net", 1) } -// testSize verifies message contnet size metadata values. +// testSize verifies message content size metadata values. func testSize(t *testing.T, store storage.Store) { mailbox := "fred" subjects := []string{"a", "br", "much longer than the others"} From fcb4bc20e0df00fb9dccb1412387d8c43f87be0a Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Mon, 22 Oct 2018 16:25:27 -0700 Subject: [PATCH 15/19] test: Add basic integration test suite, closes #119 --- go.mod | 1 + go.sum | 2 + pkg/test/integration_test.go | 173 ++++++++++++++++++++++++++++++ pkg/test/testdata/basic.golden | 12 +++ pkg/test/testdata/basic.txt | 5 + pkg/test/testdata/fullname.golden | 12 +++ pkg/test/testdata/fullname.txt | 5 + 7 files changed, 210 insertions(+) create mode 100644 pkg/test/integration_test.go create mode 100644 pkg/test/testdata/basic.golden create mode 100644 pkg/test/testdata/basic.txt create mode 100644 pkg/test/testdata/fullname.golden create mode 100644 pkg/test/testdata/fullname.txt diff --git a/go.mod b/go.mod index 4291717..8041b34 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/gorilla/sessions v1.1.3 github.com/gorilla/websocket v1.4.0 github.com/jhillyerd/enmime v0.2.1 + github.com/jhillyerd/goldiff v0.1.0 github.com/kelseyhightower/envconfig v1.3.0 github.com/microcosm-cc/bluemonday v1.0.1 github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 7b3fe17..55a81b2 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/jhillyerd/enmime v0.2.1 h1:YodBfMH3jmrZn68Gg4ZoZH1ECDsdh8BLW9+DjoFce6o= github.com/jhillyerd/enmime v0.2.1/go.mod h1:0gWUCFBL87cvx6/MSSGNBHJ6r+fMArqltDFwHxC10P4= +github.com/jhillyerd/goldiff v0.1.0 h1:7JzKPKVwAg1GzrbnsToYzq3Y5+S7dXM4hgEYiOzaf4A= +github.com/jhillyerd/goldiff v0.1.0/go.mod h1:WeDal6DTqhbMhNkf5REzWCIvKl3JWs0Q9omZ/huIWAs= github.com/kelseyhightower/envconfig v1.3.0 h1:IvRS4f2VcIQy6j4ORGIf9145T/AsUB+oY8LyvN8BXNM= github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= diff --git a/pkg/test/integration_test.go b/pkg/test/integration_test.go new file mode 100644 index 0000000..3dbca28 --- /dev/null +++ b/pkg/test/integration_test.go @@ -0,0 +1,173 @@ +package test + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + smtpclient "net/smtp" + "os" + "path/filepath" + "testing" + "time" + + "github.com/jhillyerd/goldiff" + "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/message" + "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/policy" + "github.com/jhillyerd/inbucket/pkg/rest" + "github.com/jhillyerd/inbucket/pkg/rest/client" + "github.com/jhillyerd/inbucket/pkg/server/smtp" + "github.com/jhillyerd/inbucket/pkg/server/web" + "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/storage/mem" + "github.com/jhillyerd/inbucket/pkg/webui" +) + +const ( + restBaseURL = "http://127.0.0.1:9000/" + smtpHost = "127.0.0.1:2500" +) + +func TestSuite(t *testing.T) { + stopServer, err := startServer() + if err != nil { + t.Fatal(err) + } + _ = stopServer + // defer stopServer() + + testCases := []struct { + name string + test func(*testing.T) + }{ + {"basic", testBasic}, + {"fullname", testFullname}, + } + for _, tc := range testCases { + t.Run(tc.name, tc.test) + } +} + +func testBasic(t *testing.T) { + client, err := client.New(restBaseURL) + if err != nil { + t.Fatal(err) + } + from := "fromuser@inbucket.org" + to := []string{"recipient@inbucket.org"} + input := readTestData("basic.txt") + + // Send mail + err = smtpclient.SendMail(smtpHost, nil, from, to, input) + if err != nil { + t.Fatal(err) + } + + // Confirm receipt + msg, err := client.GetMessage("recipient", "latest") + if err != nil { + t.Fatal(err) + } + if msg == nil { + t.Errorf("Got nil message, wanted non-nil message.") + } + + // Compare to golden. + got := formatMessage(msg) + goldiff.File(t, got, "testdata", "basic.golden") +} + +func testFullname(t *testing.T) { + client, err := client.New(restBaseURL) + if err != nil { + t.Fatal(err) + } + from := "fromuser@inbucket.org" + to := []string{"recipient@inbucket.org"} + input := readTestData("fullname.txt") + + // Send mail + err = smtpclient.SendMail(smtpHost, nil, from, to, input) + if err != nil { + t.Fatal(err) + } + + // Confirm receipt + msg, err := client.GetMessage("recipient", "latest") + if err != nil { + t.Fatal(err) + } + if msg == nil { + t.Errorf("Got nil message, wanted non-nil message.") + } + + // Compare to golden. + got := formatMessage(msg) + goldiff.File(t, got, "testdata", "fullname.golden") +} + +func formatMessage(m *client.Message) []byte { + b := &bytes.Buffer{} + fmt.Fprintf(b, "Mailbox: %v\n", m.Mailbox) + fmt.Fprintf(b, "From: %v\n", m.From) + fmt.Fprintf(b, "To: %v\n", m.To) + fmt.Fprintf(b, "Subject: %v\n", m.Subject) + fmt.Fprintf(b, "Size: %v\n", m.Size) + fmt.Fprintf(b, "\nBODY TEXT:\n%v\n", m.Body.Text) + fmt.Fprintf(b, "\nBODY HTML:\n%v\n", m.Body.HTML) + return b.Bytes() +} + +func startServer() (func(), error) { + // TODO Refactor inbucket/main.go so we don't need to repeat all this here. + storage.Constructors["memory"] = mem.New + os.Clearenv() + conf, err := config.Process() + if err != nil { + return nil, err + } + rootCtx, rootCancel := context.WithCancel(context.Background()) + shutdownChan := make(chan bool) + store, err := storage.FromConfig(conf.Storage) + if err != nil { + rootCancel() + return nil, err + } + msgHub := msghub.New(rootCtx, conf.Web.MonitorHistory) + addrPolicy := &policy.Addressing{Config: conf} + mmanager := &message.StoreManager{AddrPolicy: addrPolicy, Store: store, Hub: msgHub} + // Start HTTP server. + web.Initialize(conf, shutdownChan, mmanager, msgHub) + rest.SetupRoutes(web.Router.PathPrefix("/api/").Subrouter()) + webui.SetupRoutes(web.Router) + go web.Start(rootCtx) + // Start SMTP server. + smtpServer := smtp.NewServer(conf.SMTP, shutdownChan, mmanager, addrPolicy) + go smtpServer.Start(rootCtx) + + // TODO Implmement an elegant way to determine server readiness. + time.Sleep(500 * time.Millisecond) + + return func() { + // Shut everything down. + close(shutdownChan) + rootCancel() + smtpServer.Drain() + }, nil +} + +func readTestData(path ...string) []byte { + // Prefix path with testdata. + p := append([]string{"testdata"}, path...) + f, err := os.Open(filepath.Join(p...)) + if err != nil { + panic(err) + } + data, err := ioutil.ReadAll(f) + if err != nil { + panic(err) + } + return data +} diff --git a/pkg/test/testdata/basic.golden b/pkg/test/testdata/basic.golden new file mode 100644 index 0000000..014c93f --- /dev/null +++ b/pkg/test/testdata/basic.golden @@ -0,0 +1,12 @@ +Mailbox: recipient +From: +To: [] +Subject: basic subject +Size: 217 + +BODY TEXT: +Basic message. + + +BODY HTML: + diff --git a/pkg/test/testdata/basic.txt b/pkg/test/testdata/basic.txt new file mode 100644 index 0000000..1087c28 --- /dev/null +++ b/pkg/test/testdata/basic.txt @@ -0,0 +1,5 @@ +From: fromuser@inbucket.org +To: recipient@inbucket.org +Subject: basic subject + +Basic message. diff --git a/pkg/test/testdata/fullname.golden b/pkg/test/testdata/fullname.golden new file mode 100644 index 0000000..e42108f --- /dev/null +++ b/pkg/test/testdata/fullname.golden @@ -0,0 +1,12 @@ +Mailbox: recipient +From: "From User" +To: ["Rec I. Pient" ] +Subject: basic subject +Size: 246 + +BODY TEXT: +Basic message. + + +BODY HTML: + diff --git a/pkg/test/testdata/fullname.txt b/pkg/test/testdata/fullname.txt new file mode 100644 index 0000000..c595d8c --- /dev/null +++ b/pkg/test/testdata/fullname.txt @@ -0,0 +1,5 @@ +From: From User +To: "Rec I. Pient" +Subject: basic subject + +Basic message. From 30e3892cb0dd9e4d48063cae7407ca55c0db66ed Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Mon, 22 Oct 2018 18:29:03 -0700 Subject: [PATCH 16/19] webui, rest: Render UTF-8 addresses correctly, fixes #117 --- etc/swaks-tests/utf8-subject.raw | 6 ++-- pkg/rest/apiv1_controller.go | 4 +-- pkg/server/web/helpers.go | 2 ++ pkg/stringutil/utils.go | 23 +++++++++++++--- pkg/stringutil/utils_test.go | 8 ++++-- pkg/test/integration_test.go | 38 +++++++++++++++++++++++--- pkg/test/testdata/encodedheader.golden | 12 ++++++++ pkg/test/testdata/encodedheader.txt | 5 ++++ pkg/test/testdata/fullname.golden | 4 +-- ui/templates/mailbox/_show.html | 4 +-- 10 files changed, 88 insertions(+), 18 deletions(-) create mode 100644 pkg/test/testdata/encodedheader.golden create mode 100644 pkg/test/testdata/encodedheader.txt diff --git a/etc/swaks-tests/utf8-subject.raw b/etc/swaks-tests/utf8-subject.raw index 689e655..24d7094 100644 --- a/etc/swaks-tests/utf8-subject.raw +++ b/etc/swaks-tests/utf8-subject.raw @@ -1,6 +1,8 @@ Date: %DATE% -To: %TO_ADDRESS% -From: %FROM_ADDRESS% +To: %TO_ADDRESS%, + =?utf-8?B?VGVzdCBvZiDIh8myyqLIr8ihyarJtMqb?= +From: =?utf-8?q?X-=C3=A4=C3=A9=C3=9F_Y-=C3=A4=C3=A9=C3=9F?= + Subject: =?utf-8?B?VGVzdCBvZiDIh8myyqLIr8ihyarJtMqb?= Thread-Topic: =?utf-8?B?VGVzdCBvZiDIh8myyqLIr8ihyarJtMqb?= Thread-Index: Ac6+4nH7mOymA+1JRQyk2LQPe1bEcw== diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index d9795eb..09ff079 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -33,7 +33,7 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( jmessages[i] = &model.JSONMessageHeaderV1{ Mailbox: name, ID: msg.ID, - From: msg.From.String(), + From: stringutil.StringAddress(msg.From), To: stringutil.StringAddressList(msg.To), Subject: msg.Subject, Date: msg.Date, @@ -79,7 +79,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( &model.JSONMessageV1{ Mailbox: name, ID: msg.ID, - From: msg.From.String(), + From: stringutil.StringAddress(msg.From), To: stringutil.StringAddressList(msg.To), Subject: msg.Subject, Date: msg.Date, diff --git a/pkg/server/web/helpers.go b/pkg/server/web/helpers.go index ea32cb2..838950f 100644 --- a/pkg/server/web/helpers.go +++ b/pkg/server/web/helpers.go @@ -8,11 +8,13 @@ import ( "strings" "time" + "github.com/jhillyerd/inbucket/pkg/stringutil" "github.com/rs/zerolog/log" ) // TemplateFuncs declares functions made available to all templates (including partials) var TemplateFuncs = template.FuncMap{ + "address": stringutil.StringAddress, "friendlyTime": FriendlyTime, "reverse": Reverse, "stringsJoin": strings.Join, diff --git a/pkg/stringutil/utils.go b/pkg/stringutil/utils.go index afde694..cae5fbd 100644 --- a/pkg/stringutil/utils.go +++ b/pkg/stringutil/utils.go @@ -19,13 +19,28 @@ func HashMailboxName(mailbox string) string { return fmt.Sprintf("%x", h.Sum(nil)) } -// StringAddressList converts a list of addresses to a list of strings +// StringAddress converts an Address to a UTF-8 string. +func StringAddress(a *mail.Address) string { + b := &strings.Builder{} + if a != nil { + if a.Name != "" { + b.WriteString(a.Name) + b.WriteRune(' ') + } + if a.Address != "" { + b.WriteRune('<') + b.WriteString(a.Address) + b.WriteRune('>') + } + } + return b.String() +} + +// StringAddressList converts a list of addresses to a list of UTF-8 strings. func StringAddressList(addrs []*mail.Address) []string { s := make([]string, len(addrs)) for i, a := range addrs { - if a != nil { - s[i] = a.String() - } + s[i] = StringAddress(a) } return s } diff --git a/pkg/stringutil/utils_test.go b/pkg/stringutil/utils_test.go index 8c0b7bd..4bc3cc7 100644 --- a/pkg/stringutil/utils_test.go +++ b/pkg/stringutil/utils_test.go @@ -17,10 +17,14 @@ func TestHashMailboxName(t *testing.T) { func TestStringAddressList(t *testing.T) { input := []*mail.Address{ - {Name: "Fred B. Fish", Address: "fred@fish.org"}, + {Name: "Fred ß. Fish", Address: "fred@fish.org"}, {Name: "User", Address: "user@domain.org"}, + {Address: "a@b.com"}, } - want := []string{`"Fred B. Fish" `, `"User" `} + want := []string{ + `Fred ß. Fish `, + `User `, + ``} output := stringutil.StringAddressList(input) if len(output) != len(want) { t.Fatalf("Got %v strings, want: %v", len(output), len(want)) diff --git a/pkg/test/integration_test.go b/pkg/test/integration_test.go index 3dbca28..9d0963a 100644 --- a/pkg/test/integration_test.go +++ b/pkg/test/integration_test.go @@ -44,6 +44,7 @@ func TestSuite(t *testing.T) { }{ {"basic", testBasic}, {"fullname", testFullname}, + {"encodedHeader", testEncodedHeader}, } for _, tc := range testCases { t.Run(tc.name, tc.test) @@ -59,13 +60,13 @@ func testBasic(t *testing.T) { to := []string{"recipient@inbucket.org"} input := readTestData("basic.txt") - // Send mail + // Send mail. err = smtpclient.SendMail(smtpHost, nil, from, to, input) if err != nil { t.Fatal(err) } - // Confirm receipt + // Confirm receipt. msg, err := client.GetMessage("recipient", "latest") if err != nil { t.Fatal(err) @@ -88,13 +89,13 @@ func testFullname(t *testing.T) { to := []string{"recipient@inbucket.org"} input := readTestData("fullname.txt") - // Send mail + // Send mail. err = smtpclient.SendMail(smtpHost, nil, from, to, input) if err != nil { t.Fatal(err) } - // Confirm receipt + // Confirm receipt. msg, err := client.GetMessage("recipient", "latest") if err != nil { t.Fatal(err) @@ -108,6 +109,35 @@ func testFullname(t *testing.T) { goldiff.File(t, got, "testdata", "fullname.golden") } +func testEncodedHeader(t *testing.T) { + client, err := client.New(restBaseURL) + if err != nil { + t.Fatal(err) + } + from := "fromuser@inbucket.org" + to := []string{"recipient@inbucket.org"} + input := readTestData("encodedheader.txt") + + // Send mail. + err = smtpclient.SendMail(smtpHost, nil, from, to, input) + if err != nil { + t.Fatal(err) + } + + // Confirm receipt. + msg, err := client.GetMessage("recipient", "latest") + if err != nil { + t.Fatal(err) + } + if msg == nil { + t.Errorf("Got nil message, wanted non-nil message.") + } + + // Compare to golden. + got := formatMessage(msg) + goldiff.File(t, got, "testdata", "encodedheader.golden") +} + func formatMessage(m *client.Message) []byte { b := &bytes.Buffer{} fmt.Fprintf(b, "Mailbox: %v\n", m.Mailbox) diff --git a/pkg/test/testdata/encodedheader.golden b/pkg/test/testdata/encodedheader.golden new file mode 100644 index 0000000..af11c2a --- /dev/null +++ b/pkg/test/testdata/encodedheader.golden @@ -0,0 +1,12 @@ +Mailbox: recipient +From: X-äéß Y-äéß +To: [Test of ȇɲʢȯȡɪɴʛ ] +Subject: Test of ȇɲʢȯȡɪɴʛ +Size: 351 + +BODY TEXT: +Basic message. + + +BODY HTML: + diff --git a/pkg/test/testdata/encodedheader.txt b/pkg/test/testdata/encodedheader.txt new file mode 100644 index 0000000..8855f72 --- /dev/null +++ b/pkg/test/testdata/encodedheader.txt @@ -0,0 +1,5 @@ +From: =?utf-8?q?X-=C3=A4=C3=A9=C3=9F_Y-=C3=A4=C3=A9=C3=9F?= +To: =?utf-8?B?VGVzdCBvZiDIh8myyqLIr8ihyarJtMqb?= +Subject: =?utf-8?b?VGVzdCBvZiDIh8myyqLIr8ihyarJtMqb?= + +Basic message. diff --git a/pkg/test/testdata/fullname.golden b/pkg/test/testdata/fullname.golden index e42108f..b40a967 100644 --- a/pkg/test/testdata/fullname.golden +++ b/pkg/test/testdata/fullname.golden @@ -1,6 +1,6 @@ Mailbox: recipient -From: "From User" -To: ["Rec I. Pient" ] +From: From User +To: [Rec I. Pient ] Subject: basic subject Size: 246 diff --git a/ui/templates/mailbox/_show.html b/ui/templates/mailbox/_show.html index 49621f0..227ec1f 100644 --- a/ui/templates/mailbox/_show.html +++ b/ui/templates/mailbox/_show.html @@ -51,12 +51,12 @@
From:
-
{{.message.From}}
+
{{.message.From | address}}
To:
{{range $i, $addr := .message.To}} {{- if $i}},{{end}} - {{$addr -}} + {{$addr | address -}} {{end}}
Date:
From 690b19a22cda1dbbacf33bdf6638e2c2c179e380 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Wed, 31 Oct 2018 18:44:32 -0700 Subject: [PATCH 17/19] rest: Rewrite client tests using httptest server. --- pkg/rest/client/apiv1_client_test.go | 463 ++++++++++++++------------- 1 file changed, 234 insertions(+), 229 deletions(-) diff --git a/pkg/rest/client/apiv1_client_test.go b/pkg/rest/client/apiv1_client_test.go index d3c8ca7..61b54c7 100644 --- a/pkg/rest/client/apiv1_client_test.go +++ b/pkg/rest/client/apiv1_client_test.go @@ -1,183 +1,259 @@ -package client +package client_test -import "testing" +import ( + "github.com/gorilla/mux" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/jhillyerd/inbucket/pkg/rest/client" +) func TestClientV1ListMailbox(t *testing.T) { - var want, got string + // Setup. + c, router, teardown := setup() + defer teardown() - c, err := New(baseURLStr) + listHandler := &jsonHandler{json: `[ + { + "mailbox": "testbox", + "id": "1", + "from": "fromuser", + "subject": "test subject", + "date": "2013-10-15T16:12:02.231532239-07:00", + "size": 264, + "seen": true + } + ]`} + + router.Path("/api/v1/mailbox/testbox").Methods("GET").Handler(listHandler) + + // Method under test. + headers, err := c.ListMailbox("testbox") if err != nil { t.Fatal(err) } - mth := &mockHTTPClient{} - c.client = mth - // Method under test - _, _ = c.ListMailbox("testbox") + if len(headers) != 1 { + t.Fatalf("Got %v headers, want 1", len(headers)) + } + h := headers[0] - want = "GET" - got = mth.req.Method + got := h.Mailbox + want := "testbox" if got != want { - t.Errorf("req.Method == %q, want %q", got, want) + t.Errorf("Mailbox got %q, want %q", got, want) } - want = baseURLStr + "/api/v1/mailbox/testbox" - got = mth.req.URL.String() + got = h.ID + want = "1" if got != want { - t.Errorf("req.URL == %q, want %q", got, want) + t.Errorf("ID got %q, want %q", got, want) + } + + got = h.From + want = "fromuser" + if got != want { + t.Errorf("From got %q, want %q", got, want) + } + + got = h.Subject + want = "test subject" + if got != want { + t.Errorf("Subject got %q, want %q", got, want) + } + + gotTime := h.Date + wantTime := time.Date(2013, 10, 15, 16, 12, 02, 231532239, time.FixedZone("UTC-7", -7*60*60)) + if !wantTime.Equal(gotTime) { + t.Errorf("Date got %v, want %v", gotTime, wantTime) + } + + gotInt := h.Size + wantInt := int64(264) + if gotInt != wantInt { + t.Errorf("Size got %v, want %v", gotInt, wantInt) + } + + wantBool := true + gotBool := h.Seen + if gotBool != wantBool { + t.Errorf("Seen got %v, want %v", gotBool, wantBool) } } func TestClientV1GetMessage(t *testing.T) { - var want, got string + // Setup. + c, router, teardown := setup() + defer teardown() - c, err := New(baseURLStr) + messageHandler := &jsonHandler{json: `{ + "mailbox": "testbox", + "id": "20170107T224128-0000", + "from": "fromuser", + "subject": "test subject", + "date": "2013-10-15T16:12:02.231532239-07:00", + "size": 264, + "seen": true, + "body": { + "text": "Plain text", + "html": "" + } + }`} + + router.Path("/api/v1/mailbox/testbox/20170107T224128-0000").Methods("GET").Handler(messageHandler) + + // Method under test. + m, err := c.GetMessage("testbox", "20170107T224128-0000") if err != nil { t.Fatal(err) } - mth := &mockHTTPClient{} - c.client = mth - - // Method under test - _, _ = c.GetMessage("testbox", "20170107T224128-0000") - - want = "GET" - got = mth.req.Method - if got != want { - t.Errorf("req.Method == %q, want %q", got, want) + if m == nil { + t.Fatalf("message was nil, wanted a value") } - want = baseURLStr + "/api/v1/mailbox/testbox/20170107T224128-0000" - got = mth.req.URL.String() + got := m.Mailbox + want := "testbox" if got != want { - t.Errorf("req.URL == %q, want %q", got, want) + t.Errorf("Mailbox got %q, want %q", got, want) + } + + got = m.ID + want = "20170107T224128-0000" + if got != want { + t.Errorf("ID got %q, want %q", got, want) + } + + got = m.From + want = "fromuser" + if got != want { + t.Errorf("From got %q, want %q", got, want) + } + + got = m.Subject + want = "test subject" + if got != want { + t.Errorf("Subject got %q, want %q", got, want) + } + + gotTime := m.Date + wantTime := time.Date(2013, 10, 15, 16, 12, 02, 231532239, time.FixedZone("UTC-7", -7*60*60)) + if !wantTime.Equal(gotTime) { + t.Errorf("Date got %v, want %v", gotTime, wantTime) + } + + gotInt := m.Size + wantInt := int64(264) + if gotInt != wantInt { + t.Errorf("Size got %v, want %v", gotInt, wantInt) + } + + gotBool := m.Seen + wantBool := true + if gotBool != wantBool { + t.Errorf("Seen got %v, want %v", gotBool, wantBool) + } + + got = m.Body.Text + want = "Plain text" + if got != want { + t.Errorf("Body Text got %q, want %q", got, want) + } + + got = m.Body.HTML + want = "" + if got != want { + t.Errorf("Body HTML got %q, want %q", got, want) } } func TestClientV1MarkSeen(t *testing.T) { - var want, got string + // Setup. + c, router, teardown := setup() + defer teardown() - c, err := New(baseURLStr) + handler := &jsonHandler{} + router.Path("/api/v1/mailbox/testbox/20170107T224128-0000").Methods("PATCH"). + Handler(handler) + + // Method under test. + err := c.MarkSeen("testbox", "20170107T224128-0000") if err != nil { t.Fatal(err) } - mth := &mockHTTPClient{} - c.client = mth - // Method under test - _ = c.MarkSeen("testbox", "20170107T224128-0000") - - want = "PATCH" - got = mth.req.Method - if got != want { - t.Errorf("req.Method == %q, want %q", got, want) - } - - want = baseURLStr + "/api/v1/mailbox/testbox/20170107T224128-0000" - got = mth.req.URL.String() - if got != want { - t.Errorf("req.URL == %q, want %q", got, want) + if !handler.called { + t.Error("Wanted HTTP handler to be called, but it was not") } } func TestClientV1GetMessageSource(t *testing.T) { - var want, got string + // Setup. + c, router, teardown := setup() + defer teardown() - c, err := New(baseURLStr) - if err != nil { - t.Fatal(err) - } - mth := &mockHTTPClient{ - body: "message source", - } - c.client = mth + router.Path("/api/v1/mailbox/testbox/20170107T224128-0000/source").Methods("GET"). + Handler(&jsonHandler{json: `message source`}) - // Method under test + // Method under test. source, err := c.GetMessageSource("testbox", "20170107T224128-0000") if err != nil { t.Fatal(err) } - want = "GET" - got = mth.req.Method + want := "message source" + got := source.String() if got != want { - t.Errorf("req.Method == %q, want %q", got, want) - } - - want = baseURLStr + "/api/v1/mailbox/testbox/20170107T224128-0000/source" - got = mth.req.URL.String() - if got != want { - t.Errorf("req.URL == %q, want %q", got, want) - } - - want = "message source" - got = source.String() - if got != want { - t.Errorf("Source == %q, want: %q", got, want) + t.Errorf("Source got %q, want %q", got, want) } } func TestClientV1DeleteMessage(t *testing.T) { - var want, got string + // Setup. + c, router, teardown := setup() + defer teardown() - c, err := New(baseURLStr) - if err != nil { - t.Fatal(err) - } - mth := &mockHTTPClient{} - c.client = mth + handler := &jsonHandler{} + router.Path("/api/v1/mailbox/testbox/20170107T224128-0000").Methods("DELETE"). + Handler(handler) - // Method under test - err = c.DeleteMessage("testbox", "20170107T224128-0000") + // Method under test. + err := c.DeleteMessage("testbox", "20170107T224128-0000") if err != nil { t.Fatal(err) } - want = "DELETE" - got = mth.req.Method - if got != want { - t.Errorf("req.Method == %q, want %q", got, want) - } - - want = baseURLStr + "/api/v1/mailbox/testbox/20170107T224128-0000" - got = mth.req.URL.String() - if got != want { - t.Errorf("req.URL == %q, want %q", got, want) + if !handler.called { + t.Error("Wanted HTTP handler to be called, but it was not") } } func TestClientV1PurgeMailbox(t *testing.T) { - var want, got string + // Setup. + c, router, teardown := setup() + defer teardown() - c, err := New(baseURLStr) - if err != nil { - t.Fatal(err) - } - mth := &mockHTTPClient{} - c.client = mth + handler := &jsonHandler{} + router.Path("/api/v1/mailbox/testbox").Methods("DELETE").Handler(handler) - // Method under test - err = c.PurgeMailbox("testbox") + // Method under test. + err := c.PurgeMailbox("testbox") if err != nil { t.Fatal(err) } - want = "DELETE" - got = mth.req.Method - if got != want { - t.Errorf("req.Method == %q, want %q", got, want) - } - - want = baseURLStr + "/api/v1/mailbox/testbox" - got = mth.req.URL.String() - if got != want { - t.Errorf("req.URL == %q, want %q", got, want) + if !handler.called { + t.Error("Wanted HTTP handler to be called, but it was not") } } func TestClientV1MessageHeader(t *testing.T) { - var want, got string - response := `[ + // Setup. + c, router, teardown := setup() + defer teardown() + + listHandler := &jsonHandler{json: `[ { "mailbox":"mailbox1", "id":"id1", @@ -187,115 +263,52 @@ func TestClientV1MessageHeader(t *testing.T) { "size":100, "seen":true } - ]` + ]`} + router.Path("/api/v1/mailbox/testbox").Methods("GET").Handler(listHandler) - c, err := New(baseURLStr) - if err != nil { - t.Fatal(err) - } - mth := &mockHTTPClient{body: response} - c.client = mth - - // Method under test + // Method under test. headers, err := c.ListMailbox("testbox") if err != nil { t.Fatal(err) } - want = "GET" - got = mth.req.Method - if got != want { - t.Errorf("req.Method == %q, want %q", got, want) - } - - want = baseURLStr + "/api/v1/mailbox/testbox" - got = mth.req.URL.String() - if got != want { - t.Errorf("req.URL == %q, want %q", got, want) - } - if len(headers) != 1 { t.Fatalf("len(headers) == %v, want 1", len(headers)) } header := headers[0] - want = "mailbox1" - got = header.Mailbox - if got != want { - t.Errorf("Mailbox == %q, want %q", got, want) - } - - want = "id1" - got = header.ID - if got != want { - t.Errorf("ID == %q, want %q", got, want) - } - - want = "from1" - got = header.From - if got != want { - t.Errorf("From == %q, want %q", got, want) - } - - want = "subject1" - got = header.Subject - if got != want { - t.Errorf("Subject == %q, want %q", got, want) - } - - wantb := true - gotb := header.Seen - if gotb != wantb { - t.Errorf("Seen == %v, want %v", gotb, wantb) - } - - // Test MessageHeader.Delete() - mth.body = "" + // Test MessageHeader.Delete(). + handler := &jsonHandler{} + router.Path("/api/v1/mailbox/mailbox1/id1").Methods("DELETE").Handler(handler) err = header.Delete() if err != nil { t.Fatal(err) } - want = "DELETE" - got = mth.req.Method - if got != want { - t.Errorf("req.Method == %q, want %q", got, want) - } - - want = baseURLStr + "/api/v1/mailbox/mailbox1/id1" - got = mth.req.URL.String() - if got != want { - t.Errorf("req.URL == %q, want %q", got, want) - } - - // Test MessageHeader.GetSource() - mth.body = "source1" - _, err = header.GetSource() + // Test MessageHeader.GetSource(). + router.Path("/api/v1/mailbox/mailbox1/id1/source").Methods("GET"). + Handler(&jsonHandler{json: `source1`}) + buf, err := header.GetSource() if err != nil { t.Fatal(err) } - want = "GET" - got = mth.req.Method + want := "source1" + got := buf.String() if got != want { - t.Errorf("req.Method == %q, want %q", got, want) + t.Errorf("Got source %q, want %q", got, want) } - want = baseURLStr + "/api/v1/mailbox/mailbox1/id1/source" - got = mth.req.URL.String() - if got != want { - t.Errorf("req.URL == %q, want %q", got, want) - } - - // Test MessageHeader.GetMessage() - mth.body = `{ + // Test MessageHeader.GetMessage(). + messageHandler := &jsonHandler{json: `{ "mailbox":"mailbox1", "id":"id1", "from":"from1", "subject":"subject1", "date":"2017-01-01T00:00:00.000-07:00", "size":100 - }` + }`} + router.Path("/api/v1/mailbox/mailbox1/id1").Methods("GET").Handler(messageHandler) message, err := header.GetMessage() if err != nil { t.Fatal(err) @@ -304,53 +317,45 @@ func TestClientV1MessageHeader(t *testing.T) { t.Fatalf("message was nil, wanted a value") } - want = "GET" - got = mth.req.Method - if got != want { - t.Errorf("req.Method == %q, want %q", got, want) - } - - want = baseURLStr + "/api/v1/mailbox/mailbox1/id1" - got = mth.req.URL.String() - if got != want { - t.Errorf("req.URL == %q, want %q", got, want) - } - - // Test Message.Delete() - mth.body = "" + // Test Message.Delete(). err = message.Delete() if err != nil { t.Fatal(err) } - want = "DELETE" - got = mth.req.Method - if got != want { - t.Errorf("req.Method == %q, want %q", got, want) - } - - want = baseURLStr + "/api/v1/mailbox/mailbox1/id1" - got = mth.req.URL.String() - if got != want { - t.Errorf("req.URL == %q, want %q", got, want) - } - - // Test MessageHeader.GetSource() - mth.body = "source1" - _, err = message.GetSource() + // Test Message.GetSource(). + buf, err = message.GetSource() if err != nil { t.Fatal(err) } - want = "GET" - got = mth.req.Method + want = "source1" + got = buf.String() if got != want { - t.Errorf("req.Method == %q, want %q", got, want) - } - - want = baseURLStr + "/api/v1/mailbox/mailbox1/id1/source" - got = mth.req.URL.String() - if got != want { - t.Errorf("req.URL == %q, want %q", got, want) + t.Errorf("Got source %q, want %q", got, want) } } + +// setup returns a client, router and server for API testing. +func setup() (c *client.Client, router *mux.Router, teardown func()) { + router = mux.NewRouter() + server := httptest.NewServer(router) + c, err := client.New(server.URL) + if err != nil { + panic(err) + } + return c, router, func() { + server.Close() + } +} + +// jsonHandler returns the string in json when servicing a request. +type jsonHandler struct { + json string + called bool +} + +func (j *jsonHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + j.called = true + w.Write([]byte(j.json)) +} From 469132fe2fe5ff2aad0138245d45cf0de42fa446 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Wed, 31 Oct 2018 19:36:56 -0700 Subject: [PATCH 18/19] rest: Add godoc example test for client. - Update README and CHANGELOG --- CHANGELOG.md | 6 ++ README.md | 4 ++ pkg/rest/client/example_test.go | 102 ++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+) create mode 100644 pkg/rest/client/example_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 369f279..bc021a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,10 +11,16 @@ This project adheres to [Semantic Versioning](http://semver.org/). - Use Go 1.11 modules for reproducible builds. - SMTP TLS support (thanks kingforaday.) - `INBUCKET_WEB_PPROF` configuration option for performance profiling. +- Godoc example for the REST API client. ### Changed - Docker build now uses Go 1.11 and Alpine 3.8 +### Fixed +- Render UTF-8 addresses correctly in both REST API and Web UI. +- Memory storage now correctly returns the newest message when asked for ID + `latest`. + ## [v2.0.0] - 2018-05-05 diff --git a/README.md b/README.md index a76c3eb..2d829a2 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,9 @@ address and make them available via web, REST and POP3. Once compiled, Inbucket does not have any external dependencies (HTTP, SMTP, POP3 and storage are all built in). +A Go client for the REST API is available in +`github.com/jhillyerd/inbucket/pkg/rest/client` - [Go API docs] + Read more at the [Inbucket Website] ![Screenshot](http://www.inbucket.org/images/inbucket-ss1.png "Viewing a message") @@ -55,6 +58,7 @@ Inbucket is written in [Google Go] Inbucket is open source software released under the MIT License. The latest version can be found at https://github.com/jhillyerd/inbucket +[Go API docs]: https://godoc.org/github.com/jhillyerd/inbucket/pkg/rest/client [Build Status]: https://travis-ci.org/jhillyerd/inbucket [Change Log]: https://github.com/jhillyerd/inbucket/blob/master/CHANGELOG.md [CONTRIBUTING.md]: https://github.com/jhillyerd/inbucket/blob/develop/CONTRIBUTING.md diff --git a/pkg/rest/client/example_test.go b/pkg/rest/client/example_test.go new file mode 100644 index 0000000..7b36ded --- /dev/null +++ b/pkg/rest/client/example_test.go @@ -0,0 +1,102 @@ +package client_test + +import ( + "fmt" + "log" + "net/http" + "net/http/httptest" + + "github.com/gorilla/mux" + "github.com/jhillyerd/inbucket/pkg/rest/client" +) + +// Example demonstrates basic usage for the Inbucket REST client. +func Example() { + // Setup a fake Inbucket server for this example. + baseURL, teardown := exampleSetup() + defer teardown() + + // Begin by creating a new client using the base URL of your Inbucket server, i.e. + // `localhost:9000`. + restClient, err := client.New(baseURL) + if err != nil { + log.Fatal(err) + } + + // Get a slice of message headers for the mailbox named `user1`. + headers, err := restClient.ListMailbox("user1") + if err != nil { + log.Fatal(err) + } + for _, header := range headers { + fmt.Printf("ID: %v, Subject: %v\n", header.ID, header.Subject) + } + + // Get the content of the first message. + message, err := headers[0].GetMessage() + if err != nil { + log.Fatal(err) + } + fmt.Printf("\nFrom: %v\n", message.From) + fmt.Printf("Text body:\n%v", message.Body.Text) + + // Delete the second message. + err = headers[1].Delete() + if err != nil { + log.Fatal(err) + } + + // Output: + // ID: 20180107T224128-0000, Subject: First subject + // ID: 20180108T121212-0123, Subject: Second subject + // + // From: admin@inbucket.org + // Text body: + // This is the plain text body +} + +// exampleSetup creates a fake Inbucket server to power Example() below. +func exampleSetup() (baseURL string, teardown func()) { + router := mux.NewRouter() + server := httptest.NewServer(router) + + // Handle ListMailbox request. + router.HandleFunc("/api/v1/mailbox/user1", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`[ + { + "mailbox": "user1", + "id": "20180107T224128-0000", + "subject": "First subject" + }, + { + "mailbox": "user1", + "id": "20180108T121212-0123", + "subject": "Second subject" + } + ]`)) + }) + + // Handle GetMessage request. + router.HandleFunc("/api/v1/mailbox/user1/20180107T224128-0000", + func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{ + "mailbox": "user1", + "id": "20180107T224128-0000", + "from": "admin@inbucket.org", + "subject": "First subject", + "body": { + "text": "This is the plain text body" + } + }`)) + }) + + // Handle Delete request. + router.HandleFunc("/api/v1/mailbox/user1/20180108T121212-0123", + func(w http.ResponseWriter, r *http.Request) { + // Nop. + }) + + return server.URL, func() { + server.Close() + } +} From 91fea4e1fd1311240764a7fdeadc3bfe5cc06158 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Wed, 31 Oct 2018 20:06:47 -0700 Subject: [PATCH 19/19] Update CHANGELOG for beta --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc021a2..76d14cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). -## [Unreleased] +## [v2.1.0-beta1] ### Added - Use Go 1.11 modules for reproducible builds. @@ -179,7 +179,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). specific message. [Unreleased]: https://github.com/jhillyerd/inbucket/compare/master...develop -[v2.0.0]: https://github.com/jhillyerd/inbucket/compare/v2.0.0-rc1...v2.0.0 +[v2.1.0-beta1]: https://github.com/jhillyerd/inbucket/compare/v2.0.0...v2.1.0-beta1 +[v2.0.0]: https://github.com/jhillyerd/inbucket/compare/v2.0.0-rc1...v2.0.0 [v2.0.0-rc1]: https://github.com/jhillyerd/inbucket/compare/v1.3.1...v2.0.0-rc1 [v1.3.1]: https://github.com/jhillyerd/inbucket/compare/v1.3.0...v1.3.1 [v1.3.0]: https://github.com/jhillyerd/inbucket/compare/v1.2.0...v1.3.0