From a58dfc5e4fb92b8106ab9e8de10fd224ce3e31e9 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 10 Mar 2018 13:34:35 -0800 Subject: [PATCH 01/26] storage: finish renaming storage packages for #79 #69 - storage: rename DataStore to Store - file: rename types to appease linter --- cmd/inbucket/main.go | 2 +- pkg/rest/apiv1_controller.go | 6 +-- pkg/rest/apiv1_controller_test.go | 30 +++++------ pkg/rest/testutils_test.go | 6 +-- pkg/server/pop3/handler.go | 28 +++++------ pkg/server/pop3/listener.go | 4 +- pkg/server/smtp/handler.go | 2 +- pkg/server/smtp/handler_test.go | 18 +++---- pkg/server/smtp/listener.go | 12 ++--- pkg/server/web/context.go | 2 +- pkg/server/web/server.go | 4 +- pkg/storage/file/fmessage.go | 44 ++++++++-------- pkg/storage/file/fstore.go | 64 ++++++++++++------------ pkg/storage/file/fstore_test.go | 14 +++--- pkg/storage/lock.go | 2 +- pkg/storage/lock_test.go | 4 +- pkg/storage/retention.go | 6 +-- pkg/storage/retention_test.go | 2 +- pkg/storage/{datastore.go => storage.go} | 8 +-- pkg/storage/testing.go | 2 +- pkg/webui/mailbox_controller.go | 10 ++-- 21 files changed, 135 insertions(+), 135 deletions(-) rename pkg/storage/{datastore.go => storage.go} (87%) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index 0886a71..6e25e44 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -116,7 +116,7 @@ func main() { msgHub := msghub.New(rootCtx, config.GetWebConfig().MonitorHistory) // Grab our datastore - ds := filestore.DefaultFileDataStore() + ds := file.DefaultStore() // Start HTTP server web.Initialize(config.GetWebConfig(), shutdownChan, ds, msgHub) diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index abd9b0b..010bfac 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -65,7 +65,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } msg, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } @@ -150,7 +150,7 @@ func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } message, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } @@ -184,7 +184,7 @@ func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } message, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index 7f5aba7..9c4e789 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -31,7 +31,7 @@ const ( func TestRestMailboxList(t *testing.T) { // Setup - ds := &datastore.MockDataStore{} + ds := &storage.MockDataStore{} logbuf := setupWebServer(ds) // Test invalid mailbox name @@ -45,9 +45,9 @@ func TestRestMailboxList(t *testing.T) { } // Test empty mailbox - emptybox := &datastore.MockMailbox{} + emptybox := &storage.MockMailbox{} ds.On("MailboxFor", "empty").Return(emptybox, nil) - emptybox.On("GetMessages").Return([]datastore.Message{}, nil) + emptybox.On("GetMessages").Return([]storage.Message{}, nil) w, err = testRestGet(baseURL + "/mailbox/empty") expectCode = 200 @@ -59,7 +59,7 @@ func TestRestMailboxList(t *testing.T) { } // Test MailboxFor error - ds.On("MailboxFor", "error").Return(&datastore.MockMailbox{}, fmt.Errorf("Internal error")) + ds.On("MailboxFor", "error").Return(&storage.MockMailbox{}, fmt.Errorf("Internal error")) w, err = testRestGet(baseURL + "/mailbox/error") expectCode = 500 if err != nil { @@ -77,9 +77,9 @@ func TestRestMailboxList(t *testing.T) { } // Test MailboxFor error - error2box := &datastore.MockMailbox{} + error2box := &storage.MockMailbox{} ds.On("MailboxFor", "error2").Return(error2box, nil) - error2box.On("GetMessages").Return([]datastore.Message{}, fmt.Errorf("Internal error 2")) + error2box.On("GetMessages").Return([]storage.Message{}, fmt.Errorf("Internal error 2")) w, err = testRestGet(baseURL + "/mailbox/error2") expectCode = 500 @@ -107,11 +107,11 @@ func TestRestMailboxList(t *testing.T) { Subject: "subject 2", Date: time.Date(2012, 7, 1, 10, 11, 12, 253, time.FixedZone("PDT", -700)), } - goodbox := &datastore.MockMailbox{} + goodbox := &storage.MockMailbox{} ds.On("MailboxFor", "good").Return(goodbox, nil) msg1 := data1.MockMessage() msg2 := data2.MockMessage() - goodbox.On("GetMessages").Return([]datastore.Message{msg1, msg2}, nil) + goodbox.On("GetMessages").Return([]storage.Message{msg1, msg2}, nil) // Check return code w, err = testRestGet(baseURL + "/mailbox/good") @@ -155,7 +155,7 @@ func TestRestMailboxList(t *testing.T) { func TestRestMessage(t *testing.T) { // Setup - ds := &datastore.MockDataStore{} + ds := &storage.MockDataStore{} logbuf := setupWebServer(ds) // Test invalid mailbox name @@ -169,9 +169,9 @@ func TestRestMessage(t *testing.T) { } // Test requesting a message that does not exist - emptybox := &datastore.MockMailbox{} + emptybox := &storage.MockMailbox{} ds.On("MailboxFor", "empty").Return(emptybox, nil) - emptybox.On("GetMessage", "0001").Return(&datastore.MockMessage{}, datastore.ErrNotExist) + emptybox.On("GetMessage", "0001").Return(&storage.MockMessage{}, storage.ErrNotExist) w, err = testRestGet(baseURL + "/mailbox/empty/0001") expectCode = 404 @@ -183,7 +183,7 @@ func TestRestMessage(t *testing.T) { } // Test MailboxFor error - ds.On("MailboxFor", "error").Return(&datastore.MockMailbox{}, fmt.Errorf("Internal error")) + ds.On("MailboxFor", "error").Return(&storage.MockMailbox{}, fmt.Errorf("Internal error")) w, err = testRestGet(baseURL + "/mailbox/error/0001") expectCode = 500 if err != nil { @@ -201,9 +201,9 @@ func TestRestMessage(t *testing.T) { } // Test GetMessage error - error2box := &datastore.MockMailbox{} + error2box := &storage.MockMailbox{} ds.On("MailboxFor", "error2").Return(error2box, nil) - error2box.On("GetMessage", "0001").Return(&datastore.MockMessage{}, fmt.Errorf("Internal error 2")) + error2box.On("GetMessage", "0001").Return(&storage.MockMessage{}, fmt.Errorf("Internal error 2")) w, err = testRestGet(baseURL + "/mailbox/error2/0001") expectCode = 500 @@ -228,7 +228,7 @@ func TestRestMessage(t *testing.T) { Text: "This is some text", HTML: "This is some HTML", } - goodbox := &datastore.MockMailbox{} + goodbox := &storage.MockMailbox{} ds.On("MailboxFor", "good").Return(goodbox, nil) msg1 := data1.MockMessage() goodbox.On("GetMessage", "0001").Return(msg1, nil) diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index 7955828..ef294df 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -25,8 +25,8 @@ type InputMessageData struct { HTML, Text string } -func (d *InputMessageData) MockMessage() *datastore.MockMessage { - msg := &datastore.MockMessage{} +func (d *InputMessageData) MockMessage() *storage.MockMessage { + msg := &storage.MockMessage{} msg.On("ID").Return(d.ID) msg.On("From").Return(d.From) msg.On("To").Return(d.To) @@ -188,7 +188,7 @@ func testRestGet(url string) (*httptest.ResponseRecorder, error) { return w, nil } -func setupWebServer(ds datastore.DataStore) *bytes.Buffer { +func setupWebServer(ds storage.Store) *bytes.Buffer { // Capture log output buf := new(bytes.Buffer) log.SetOutput(buf) diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index 7922ca2..98cce65 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -57,18 +57,18 @@ var commands = map[string]bool{ // Session defines an active POP3 session type Session struct { - server *Server // Reference to the server we belong to - id int // Session ID number - conn net.Conn // Our network connection - remoteHost string // IP address of client - sendError error // Used to bail out of read loop on send error - state State // Current session state - reader *bufio.Reader // Buffered reader for our net conn - user string // Mailbox name - mailbox datastore.Mailbox // Mailbox instance - messages []datastore.Message // Slice of messages in mailbox - retain []bool // Messages to retain upon UPDATE (true=retain) - msgCount int // Number of undeleted messages + server *Server // Reference to the server we belong to + id int // Session ID number + conn net.Conn // Our network connection + remoteHost string // IP address of client + sendError error // Used to bail out of read loop on send error + state State // Current session state + reader *bufio.Reader // Buffered reader for our net conn + user string // Mailbox name + mailbox storage.Mailbox // Mailbox instance + messages []storage.Message // Slice of messages in mailbox + retain []bool // Messages to retain upon UPDATE (true=retain) + msgCount int // Number of undeleted messages } // NewSession creates a new POP3 session @@ -432,7 +432,7 @@ func (ses *Session) transactionHandler(cmd string, args []string) { } // Send the contents of the message to the client -func (ses *Session) sendMessage(msg datastore.Message) { +func (ses *Session) sendMessage(msg storage.Message) { reader, err := msg.RawReader() if err != nil { ses.logError("Failed to read message for RETR command") @@ -465,7 +465,7 @@ func (ses *Session) sendMessage(msg datastore.Message) { } // Send the headers plus the top N lines to the client -func (ses *Session) sendMessageTop(msg datastore.Message, lineCount int) { +func (ses *Session) sendMessageTop(msg storage.Message, lineCount int) { reader, err := msg.RawReader() if err != nil { ses.logError("Failed to read message for RETR command") diff --git a/pkg/server/pop3/listener.go b/pkg/server/pop3/listener.go index 42aaa56..c971854 100644 --- a/pkg/server/pop3/listener.go +++ b/pkg/server/pop3/listener.go @@ -17,14 +17,14 @@ type Server struct { host string domain string maxIdleSeconds int - dataStore datastore.DataStore + dataStore storage.Store listener net.Listener globalShutdown chan bool waitgroup *sync.WaitGroup } // New creates a new Server struct -func New(cfg config.POP3Config, shutdownChan chan bool, ds datastore.DataStore) *Server { +func New(cfg config.POP3Config, shutdownChan chan bool, ds storage.Store) *Server { return &Server{ host: fmt.Sprintf("%v:%v", cfg.IP4address, cfg.IP4port), domain: cfg.Domain, diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 96646b1..a678d3a 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -73,7 +73,7 @@ var commands = map[string]bool{ // recipientDetails for message delivery type recipientDetails struct { address, localPart, domainPart string - mailbox datastore.Mailbox + mailbox storage.Mailbox } // Session holds the state of an SMTP session diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index c1e6168..b53b651 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -26,7 +26,7 @@ type scriptStep struct { // Test commands in GREET state func TestGreetState(t *testing.T) { // Setup mock objects - mds := &datastore.MockDataStore{} + mds := &storage.MockDataStore{} server, logbuf, teardown := setupSMTPServer(mds) defer teardown() @@ -83,7 +83,7 @@ func TestGreetState(t *testing.T) { // Test commands in READY state func TestReadyState(t *testing.T) { // Setup mock objects - mds := &datastore.MockDataStore{} + mds := &storage.MockDataStore{} server, logbuf, teardown := setupSMTPServer(mds) defer teardown() @@ -144,9 +144,9 @@ func TestReadyState(t *testing.T) { // Test commands in MAIL state func TestMailState(t *testing.T) { // Setup mock objects - mds := &datastore.MockDataStore{} - mb1 := &datastore.MockMailbox{} - msg1 := &datastore.MockMessage{} + mds := &storage.MockDataStore{} + mb1 := &storage.MockMailbox{} + msg1 := &storage.MockMessage{} mds.On("MailboxFor", "u1").Return(mb1, nil) mb1.On("NewMessage").Return(msg1, nil) mb1.On("Name").Return("u1") @@ -259,9 +259,9 @@ func TestMailState(t *testing.T) { // Test commands in DATA state func TestDataState(t *testing.T) { // Setup mock objects - mds := &datastore.MockDataStore{} - mb1 := &datastore.MockMailbox{} - msg1 := &datastore.MockMessage{} + mds := &storage.MockDataStore{} + mb1 := &storage.MockMailbox{} + msg1 := &storage.MockMessage{} mds.On("MailboxFor", "u1").Return(mb1, nil) mb1.On("NewMessage").Return(msg1, nil) mb1.On("Name").Return("u1") @@ -367,7 +367,7 @@ func (m *mockConn) SetDeadline(t time.Time) error { return nil } func (m *mockConn) SetReadDeadline(t time.Time) error { return nil } func (m *mockConn) SetWriteDeadline(t time.Time) error { return nil } -func setupSMTPServer(ds datastore.DataStore) (s *Server, buf *bytes.Buffer, teardown func()) { +func setupSMTPServer(ds storage.Store) (s *Server, buf *bytes.Buffer, teardown func()) { // Test Server Config cfg := config.SMTPConfig{ IP4address: net.IPv4(127, 0, 0, 1), diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index 986d112..4586174 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -48,10 +48,10 @@ type Server struct { storeMessages bool // Dependencies - dataStore datastore.DataStore // Mailbox/message store - globalShutdown chan bool // Shuts down Inbucket - msgHub *msghub.Hub // Pub/sub for message info - retentionScanner *datastore.RetentionScanner // Deletes expired messages + dataStore storage.Store // Mailbox/message store + globalShutdown chan bool // Shuts down Inbucket + msgHub *msghub.Hub // Pub/sub for message info + retentionScanner *storage.RetentionScanner // Deletes expired messages // State listener net.Listener // Incoming network connections @@ -83,7 +83,7 @@ var ( func NewServer( cfg config.SMTPConfig, globalShutdown chan bool, - ds datastore.DataStore, + ds storage.Store, msgHub *msghub.Hub) *Server { return &Server{ host: fmt.Sprintf("%v:%v", cfg.IP4address, cfg.IP4port), @@ -96,7 +96,7 @@ func NewServer( globalShutdown: globalShutdown, dataStore: ds, msgHub: msgHub, - retentionScanner: datastore.NewRetentionScanner(ds, globalShutdown), + retentionScanner: storage.NewRetentionScanner(ds, globalShutdown), waitgroup: new(sync.WaitGroup), } } diff --git a/pkg/server/web/context.go b/pkg/server/web/context.go index 01d6c85..422fdb5 100644 --- a/pkg/server/web/context.go +++ b/pkg/server/web/context.go @@ -15,7 +15,7 @@ import ( type Context struct { Vars map[string]string Session *sessions.Session - DataStore datastore.DataStore + DataStore storage.Store MsgHub *msghub.Hub WebConfig config.WebConfig IsJSON bool diff --git a/pkg/server/web/server.go b/pkg/server/web/server.go index a9cd22a..5c82430 100644 --- a/pkg/server/web/server.go +++ b/pkg/server/web/server.go @@ -23,7 +23,7 @@ type Handler func(http.ResponseWriter, *http.Request, *Context) error var ( // DataStore is where all the mailboxes and messages live - DataStore datastore.DataStore + DataStore storage.Store // msgHub holds a reference to the message pub/sub system msgHub *msghub.Hub @@ -51,7 +51,7 @@ func init() { func Initialize( cfg config.WebConfig, shutdownChan chan bool, - ds datastore.DataStore, + ds storage.Store, mh *msghub.Hub) { webConfig = cfg diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index 22de241..4e496f8 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -1,4 +1,4 @@ -package filestore +package file import ( "bufio" @@ -15,10 +15,10 @@ import ( "github.com/jhillyerd/inbucket/pkg/storage" ) -// FileMessage implements Message and contains a little bit of data about a +// Message implements Message and contains a little bit of data about a // particular email message, and methods to retrieve the rest of it from disk. -type FileMessage struct { - mailbox *FileMailbox +type Message struct { + mailbox *Mailbox // Stored in GOB Fid string Fdate time.Time @@ -34,7 +34,7 @@ type FileMessage struct { // NewMessage creates a new FileMessage object and sets the Date and Id fields. // It will also delete messages over messageCap if configured. -func (mb *FileMailbox) NewMessage() (datastore.Message, error) { +func (mb *Mailbox) NewMessage() (storage.Message, error) { // Load index if !mb.indexLoaded { if err := mb.readIndex(); err != nil { @@ -54,50 +54,50 @@ func (mb *FileMailbox) NewMessage() (datastore.Message, error) { date := time.Now() id := generateID(date) - return &FileMessage{mailbox: mb, Fid: id, Fdate: date, writable: true}, nil + return &Message{mailbox: mb, Fid: id, Fdate: date, writable: true}, nil } // ID gets the ID of the Message -func (m *FileMessage) ID() string { +func (m *Message) ID() string { return m.Fid } // Date returns the date/time this Message was received by Inbucket -func (m *FileMessage) Date() time.Time { +func (m *Message) Date() time.Time { return m.Fdate } // From returns the value of the Message From header -func (m *FileMessage) From() string { +func (m *Message) From() string { return m.Ffrom } // To returns the value of the Message To header -func (m *FileMessage) To() []string { +func (m *Message) To() []string { return m.Fto } // Subject returns the value of the Message Subject header -func (m *FileMessage) Subject() string { +func (m *Message) Subject() string { return m.Fsubject } // String returns a string in the form: "Subject()" from From() -func (m *FileMessage) String() string { +func (m *Message) String() string { return fmt.Sprintf("\"%v\" from %v", m.Fsubject, m.Ffrom) } // Size returns the size of the Message on disk in bytes -func (m *FileMessage) Size() int64 { +func (m *Message) Size() int64 { return m.Fsize } -func (m *FileMessage) rawPath() string { +func (m *Message) rawPath() string { return filepath.Join(m.mailbox.path, m.Fid+".raw") } // ReadHeader opens the .raw portion of a Message and returns a standard Go mail.Message object -func (m *FileMessage) ReadHeader() (msg *mail.Message, err error) { +func (m *Message) ReadHeader() (msg *mail.Message, err error) { file, err := os.Open(m.rawPath()) if err != nil { return nil, err @@ -113,7 +113,7 @@ func (m *FileMessage) ReadHeader() (msg *mail.Message, err error) { } // ReadBody opens the .raw portion of a Message and returns a MIMEBody object -func (m *FileMessage) ReadBody() (body *enmime.Envelope, err error) { +func (m *Message) ReadBody() (body *enmime.Envelope, err error) { file, err := os.Open(m.rawPath()) if err != nil { return nil, err @@ -133,7 +133,7 @@ func (m *FileMessage) ReadBody() (body *enmime.Envelope, err error) { } // RawReader opens the .raw portion of a Message as an io.ReadCloser -func (m *FileMessage) RawReader() (reader io.ReadCloser, err error) { +func (m *Message) RawReader() (reader io.ReadCloser, err error) { file, err := os.Open(m.rawPath()) if err != nil { return nil, err @@ -142,7 +142,7 @@ func (m *FileMessage) RawReader() (reader io.ReadCloser, err error) { } // ReadRaw opens the .raw portion of a Message and returns it as a string -func (m *FileMessage) ReadRaw() (raw *string, err error) { +func (m *Message) ReadRaw() (raw *string, err error) { reader, err := m.RawReader() if err != nil { return nil, err @@ -163,10 +163,10 @@ func (m *FileMessage) ReadRaw() (raw *string, err error) { // Append data to a newly opened Message, this will fail on a pre-existing Message and // after Close() is called. -func (m *FileMessage) Append(data []byte) error { +func (m *Message) Append(data []byte) error { // Prevent Appending to a pre-existing Message if !m.writable { - return datastore.ErrNotWritable + return storage.ErrNotWritable } // Open file for writing if we haven't yet if m.writer == nil { @@ -190,7 +190,7 @@ func (m *FileMessage) Append(data []byte) error { // Close this Message for writing - no more data may be Appended. Close() will also // trigger the creation of the .gob file. -func (m *FileMessage) Close() error { +func (m *Message) Close() error { // nil out the writer fields so they can't be used writer := m.writer writerFile := m.writerFile @@ -245,7 +245,7 @@ func (m *FileMessage) Close() error { // Delete this Message from disk by removing it from the index and deleting the // raw files. -func (m *FileMessage) Delete() error { +func (m *Message) Delete() error { messages := m.mailbox.messages for i, mm := range messages { if m == mm { diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index 7d7dd35..5588cbb 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -1,4 +1,4 @@ -package filestore +package file import ( "bufio" @@ -48,17 +48,17 @@ func countGenerator(c chan int) { } } -// FileDataStore implements DataStore aand is the root of the mail storage +// Store implements DataStore aand is the root of the mail storage // hiearchy. It provides access to Mailbox objects -type FileDataStore struct { - hashLock datastore.HashLock +type Store struct { + hashLock storage.HashLock path string mailPath string messageCap int } -// NewFileDataStore creates a new DataStore object using the specified path -func NewFileDataStore(cfg config.DataStoreConfig) datastore.DataStore { +// New creates a new DataStore object using the specified path +func New(cfg config.DataStoreConfig) storage.Store { path := cfg.Path if path == "" { log.Errorf("No value configured for datastore path") @@ -71,19 +71,19 @@ func NewFileDataStore(cfg config.DataStoreConfig) datastore.DataStore { log.Errorf("Error creating dir %q: %v", mailPath, err) } } - return &FileDataStore{path: path, mailPath: mailPath, messageCap: cfg.MailboxMsgCap} + return &Store{path: path, mailPath: mailPath, messageCap: cfg.MailboxMsgCap} } -// DefaultFileDataStore creates a new DataStore object. It uses the inbucket.Config object to +// DefaultStore creates a new DataStore object. It uses the inbucket.Config object to // construct it's path. -func DefaultFileDataStore() datastore.DataStore { +func DefaultStore() storage.Store { cfg := config.GetDataStoreConfig() - return NewFileDataStore(cfg) + return New(cfg) } // MailboxFor retrieves the Mailbox object for a specified email address, if the mailbox // does not exist, it will attempt to create it. -func (ds *FileDataStore) MailboxFor(emailAddress string) (datastore.Mailbox, error) { +func (ds *Store) MailboxFor(emailAddress string) (storage.Mailbox, error) { name, err := stringutil.ParseMailboxName(emailAddress) if err != nil { return nil, err @@ -94,13 +94,13 @@ func (ds *FileDataStore) MailboxFor(emailAddress string) (datastore.Mailbox, err path := filepath.Join(ds.mailPath, s1, s2, dir) indexPath := filepath.Join(path, indexFileName) - return &FileMailbox{store: ds, name: name, dirName: dir, path: path, + return &Mailbox{store: ds, name: name, dirName: dir, path: path, indexPath: indexPath}, nil } // AllMailboxes returns a slice with all Mailboxes -func (ds *FileDataStore) AllMailboxes() ([]datastore.Mailbox, error) { - mailboxes := make([]datastore.Mailbox, 0, 100) +func (ds *Store) AllMailboxes() ([]storage.Mailbox, error) { + mailboxes := make([]storage.Mailbox, 0, 100) infos1, err := ioutil.ReadDir(ds.mailPath) if err != nil { return nil, err @@ -127,7 +127,7 @@ func (ds *FileDataStore) AllMailboxes() ([]datastore.Mailbox, error) { mbdir := inf3.Name() mbpath := filepath.Join(ds.mailPath, l1, l2, mbdir) idx := filepath.Join(mbpath, indexFileName) - mb := &FileMailbox{store: ds, dirName: mbdir, path: mbpath, + mb := &Mailbox{store: ds, dirName: mbdir, path: mbpath, indexPath: idx} mailboxes = append(mailboxes, mb) } @@ -141,7 +141,7 @@ func (ds *FileDataStore) AllMailboxes() ([]datastore.Mailbox, error) { } // LockFor returns the RWMutex for this mailbox, or an error. -func (ds *FileDataStore) LockFor(emailAddress string) (*sync.RWMutex, error) { +func (ds *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { name, err := stringutil.ParseMailboxName(emailAddress) if err != nil { return nil, err @@ -150,38 +150,38 @@ func (ds *FileDataStore) LockFor(emailAddress string) (*sync.RWMutex, error) { return ds.hashLock.Get(hash), nil } -// FileMailbox implements Mailbox, manages the mail for a specific user and +// Mailbox implements Mailbox, manages the mail for a specific user and // correlates to a particular directory on disk. -type FileMailbox struct { - store *FileDataStore +type Mailbox struct { + store *Store name string dirName string path string indexLoaded bool indexPath string - messages []*FileMessage + messages []*Message } // Name of the mailbox -func (mb *FileMailbox) Name() string { +func (mb *Mailbox) Name() string { return mb.name } // String renders the name and directory path of the mailbox -func (mb *FileMailbox) String() string { +func (mb *Mailbox) String() string { return mb.name + "[" + mb.dirName + "]" } // GetMessages scans the mailbox directory for .gob files and decodes them into // a slice of Message objects. -func (mb *FileMailbox) GetMessages() ([]datastore.Message, error) { +func (mb *Mailbox) GetMessages() ([]storage.Message, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err } } - messages := make([]datastore.Message, len(mb.messages)) + messages := make([]storage.Message, len(mb.messages)) for i, m := range mb.messages { messages[i] = m } @@ -189,7 +189,7 @@ func (mb *FileMailbox) GetMessages() ([]datastore.Message, error) { } // GetMessage decodes a single message by Id and returns a Message object -func (mb *FileMailbox) GetMessage(id string) (datastore.Message, error) { +func (mb *Mailbox) GetMessage(id string) (storage.Message, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err @@ -206,17 +206,17 @@ func (mb *FileMailbox) GetMessage(id string) (datastore.Message, error) { } } - return nil, datastore.ErrNotExist + return nil, storage.ErrNotExist } // Purge deletes all messages in this mailbox -func (mb *FileMailbox) Purge() error { +func (mb *Mailbox) Purge() error { mb.messages = mb.messages[:0] return mb.writeIndex() } // readIndex loads the mailbox index data from disk -func (mb *FileMailbox) readIndex() error { +func (mb *Mailbox) readIndex() error { // Clear message slice, open index mb.messages = mb.messages[:0] // Lock for reading @@ -242,7 +242,7 @@ func (mb *FileMailbox) readIndex() error { // Decode gob data dec := gob.NewDecoder(bufio.NewReader(file)) for { - msg := new(FileMessage) + msg := new(Message) if err = dec.Decode(msg); err != nil { if err == io.EOF { // It's OK to get an EOF here @@ -259,7 +259,7 @@ func (mb *FileMailbox) readIndex() error { } // writeIndex overwrites the index on disk with the current mailbox data -func (mb *FileMailbox) writeIndex() error { +func (mb *Mailbox) writeIndex() error { // Lock for writing indexMx.Lock() defer indexMx.Unlock() @@ -301,7 +301,7 @@ func (mb *FileMailbox) writeIndex() error { } // createDir checks for the presence of the path for this mailbox, creates it if needed -func (mb *FileMailbox) createDir() error { +func (mb *Mailbox) createDir() error { dirMx.Lock() defer dirMx.Unlock() if _, err := os.Stat(mb.path); err != nil { @@ -314,7 +314,7 @@ func (mb *FileMailbox) createDir() error { } // removeDir removes the mailbox, plus empty higher level directories -func (mb *FileMailbox) removeDir() error { +func (mb *Mailbox) removeDir() error { dirMx.Lock() defer dirMx.Unlock() // remove mailbox dir, including index file diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index b9fd41c..b02bdd1 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -1,4 +1,4 @@ -package filestore +package file import ( "bytes" @@ -359,7 +359,7 @@ func TestFSMissing(t *testing.T) { // Delete a message file without removing it from index msg, err := mb.GetMessage(sentIds[1]) assert.Nil(t, err) - fmsg := msg.(*FileMessage) + fmsg := msg.(*Message) _ = os.Remove(fmsg.rawPath()) msg, err = mb.GetMessage(sentIds[1]) assert.Nil(t, err) @@ -508,7 +508,7 @@ func TestGetLatestMessage(t *testing.T) { } // setupDataStore creates a new FileDataStore in a temporary directory -func setupDataStore(cfg config.DataStoreConfig) (*FileDataStore, *bytes.Buffer) { +func setupDataStore(cfg config.DataStoreConfig) (*Store, *bytes.Buffer) { path, err := ioutil.TempDir("", "inbucket") if err != nil { panic(err) @@ -519,12 +519,12 @@ func setupDataStore(cfg config.DataStoreConfig) (*FileDataStore, *bytes.Buffer) log.SetOutput(buf) cfg.Path = path - return NewFileDataStore(cfg).(*FileDataStore), buf + return New(cfg).(*Store), buf } // deliverMessage creates and delivers a message to the specific mailbox, returning // the size of the generated message. -func deliverMessage(ds *FileDataStore, mbName string, subject string, +func deliverMessage(ds *Store, mbName string, subject string, date time.Time) (id string, size int64) { // Build fake SMTP message for delivery testMsg := make([]byte, 0, 300) @@ -544,7 +544,7 @@ func deliverMessage(ds *FileDataStore, mbName string, subject string, if err != nil { panic(err) } - fmsg := msg.(*FileMessage) + fmsg := msg.(*Message) fmsg.Fdate = date fmsg.Fid = id if err = msg.Append(testMsg); err != nil { @@ -557,7 +557,7 @@ func deliverMessage(ds *FileDataStore, mbName string, subject string, return id, int64(len(testMsg)) } -func teardownDataStore(ds *FileDataStore) { +func teardownDataStore(ds *Store) { if err := os.RemoveAll(ds.path); err != nil { panic(err) } diff --git a/pkg/storage/lock.go b/pkg/storage/lock.go index 5247702..8612e80 100644 --- a/pkg/storage/lock.go +++ b/pkg/storage/lock.go @@ -1,4 +1,4 @@ -package datastore +package storage import ( "strconv" diff --git a/pkg/storage/lock_test.go b/pkg/storage/lock_test.go index 203ebf1..ab87bba 100644 --- a/pkg/storage/lock_test.go +++ b/pkg/storage/lock_test.go @@ -1,4 +1,4 @@ -package datastore_test +package storage_test import ( "testing" @@ -7,7 +7,7 @@ import ( ) func TestHashLock(t *testing.T) { - hl := &datastore.HashLock{} + hl := &storage.HashLock{} // Invalid hashes testCases := []struct { diff --git a/pkg/storage/retention.go b/pkg/storage/retention.go index ef3154b..f067843 100644 --- a/pkg/storage/retention.go +++ b/pkg/storage/retention.go @@ -1,4 +1,4 @@ -package datastore +package storage import ( "container/list" @@ -47,14 +47,14 @@ func init() { type RetentionScanner struct { globalShutdown chan bool // Closes when Inbucket needs to shut down retentionShutdown chan bool // Closed after the scanner has shut down - ds DataStore + ds Store retentionPeriod time.Duration retentionSleep time.Duration } // NewRetentionScanner launches a go-routine that scans for expired // messages, following the configured interval -func NewRetentionScanner(ds DataStore, shutdownChannel chan bool) *RetentionScanner { +func NewRetentionScanner(ds Store, shutdownChannel chan bool) *RetentionScanner { cfg := config.GetDataStoreConfig() rs := &RetentionScanner{ globalShutdown: shutdownChannel, diff --git a/pkg/storage/retention_test.go b/pkg/storage/retention_test.go index c357f7e..ae221e9 100644 --- a/pkg/storage/retention_test.go +++ b/pkg/storage/retention_test.go @@ -1,4 +1,4 @@ -package datastore +package storage import ( "fmt" diff --git a/pkg/storage/datastore.go b/pkg/storage/storage.go similarity index 87% rename from pkg/storage/datastore.go rename to pkg/storage/storage.go index a9bcb57..a137f9b 100644 --- a/pkg/storage/datastore.go +++ b/pkg/storage/storage.go @@ -1,5 +1,5 @@ -// Package datastore contains implementation independent datastore logic -package datastore +// Package storage contains implementation independent datastore logic +package storage import ( "errors" @@ -19,8 +19,8 @@ var ( ErrNotWritable = errors.New("Message not writable") ) -// DataStore is an interface to get Mailboxes stored in Inbucket -type DataStore interface { +// Store is an interface to get Mailboxes stored in Inbucket +type Store interface { MailboxFor(emailAddress string) (Mailbox, error) AllMailboxes() ([]Mailbox, error) // LockFor is a temporary hack to fix #77 until Datastore revamp diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index aa6c3de..fc8dbab 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -1,4 +1,4 @@ -package datastore +package storage import ( "io" diff --git a/pkg/webui/mailbox_controller.go b/pkg/webui/mailbox_controller.go index bc2ef77..f9684b9 100644 --- a/pkg/webui/mailbox_controller.go +++ b/pkg/webui/mailbox_controller.go @@ -105,7 +105,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } msg, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } @@ -154,7 +154,7 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } message, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } @@ -191,7 +191,7 @@ func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } message, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } @@ -237,7 +237,7 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Co return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } message, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } @@ -290,7 +290,7 @@ func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Contex return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) } message, err := mb.GetMessage(id) - if err == datastore.ErrNotExist { + if err == storage.ErrNotExist { http.NotFound(w, req) return nil } From 9c18f1fb30e69e077a7ca65f9e96ee118e828521 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 10 Mar 2018 18:50:18 -0800 Subject: [PATCH 02/26] Large refactor for #69 - makefile: Don't refresh deps automatically, causes double build - storage: Move GetMessage, GetMessages (Mailbox), PurgeMessages to the Store API for #69 - storage: Remove Mailbox.Name method for #69 - test: Create new test package for #79 - test: Implement StoreStub, migrate some tests off MockDataStore for task #80 - rest & webui: update controllers to use new Store methods --- .travis.yml | 1 + Makefile | 4 +- pkg/rest/apiv1_controller.go | 35 ++---------- pkg/rest/apiv1_controller_test.go | 71 ++++-------------------- pkg/server/pop3/handler.go | 7 +-- pkg/server/smtp/handler.go | 7 ++- pkg/storage/file/fstore.go | 49 +++++++++++++---- pkg/storage/file/fstore_test.go | 91 +++++++------------------------ pkg/storage/storage.go | 6 +- pkg/storage/testing.go | 24 ++++++-- pkg/test/storage.go | 47 ++++++++++++++++ pkg/webui/mailbox_controller.go | 45 +++------------ 12 files changed, 160 insertions(+), 227 deletions(-) create mode 100644 pkg/test/storage.go diff --git a/.travis.yml b/.travis.yml index c820ee3..817ab3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ env: before_script: - go get github.com/golang/lint/golint + - make deps go: - "1.10" diff --git a/Makefile b/Makefile index c5104cc..821c674 100644 --- a/Makefile +++ b/Makefile @@ -19,9 +19,9 @@ clean: deps: go get -t ./... -build: deps $(commands) +build: $(commands) -test: deps +test: go test -race ./... fmt: diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index 010bfac..9d7d1c6 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -24,12 +24,7 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - messages, err := mb.GetMessages() + messages, err := ctx.DataStore.GetMessages(name) if err != nil { // This doesn't indicate empty, likely an IO error return fmt.Errorf("Failed to get messages for %v: %v", name, err) @@ -59,12 +54,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - msg, err := mb.GetMessage(id) + msg, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -121,13 +111,8 @@ func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } // Delete all messages - err = mb.Purge() + err = ctx.DataStore.PurgeMessages(name) if err != nil { return fmt.Errorf("Mailbox(%q) purge failed: %v", name, err) } @@ -144,12 +129,7 @@ func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - message, err := mb.GetMessage(id) + message, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -178,12 +158,7 @@ func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - message, err := mb.GetMessage(id) + message, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index 9c4e789..a48fd38 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -2,14 +2,13 @@ package rest import ( "encoding/json" - "fmt" "io" "net/mail" "os" "testing" "time" - "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/test" ) const ( @@ -31,7 +30,7 @@ const ( func TestRestMailboxList(t *testing.T) { // Setup - ds := &storage.MockDataStore{} + ds := test.NewStore() logbuf := setupWebServer(ds) // Test invalid mailbox name @@ -45,10 +44,6 @@ func TestRestMailboxList(t *testing.T) { } // Test empty mailbox - emptybox := &storage.MockMailbox{} - ds.On("MailboxFor", "empty").Return(emptybox, nil) - emptybox.On("GetMessages").Return([]storage.Message{}, nil) - w, err = testRestGet(baseURL + "/mailbox/empty") expectCode = 200 if err != nil { @@ -58,30 +53,8 @@ func TestRestMailboxList(t *testing.T) { t.Errorf("Expected code %v, got %v", expectCode, w.Code) } - // Test MailboxFor error - ds.On("MailboxFor", "error").Return(&storage.MockMailbox{}, fmt.Errorf("Internal error")) - w, err = testRestGet(baseURL + "/mailbox/error") - expectCode = 500 - if err != nil { - t.Fatal(err) - } - if w.Code != expectCode { - t.Errorf("Expected code %v, got %v", expectCode, w.Code) - } - - if t.Failed() { - // Wait for handler to finish logging - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure - _, _ = io.Copy(os.Stderr, logbuf) - } - - // Test MailboxFor error - error2box := &storage.MockMailbox{} - ds.On("MailboxFor", "error2").Return(error2box, nil) - error2box.On("GetMessages").Return([]storage.Message{}, fmt.Errorf("Internal error 2")) - - w, err = testRestGet(baseURL + "/mailbox/error2") + // Test Mailbox error + w, err = testRestGet(baseURL + "/mailbox/messageserr") expectCode = 500 if err != nil { t.Fatal(err) @@ -107,11 +80,10 @@ func TestRestMailboxList(t *testing.T) { Subject: "subject 2", Date: time.Date(2012, 7, 1, 10, 11, 12, 253, time.FixedZone("PDT", -700)), } - goodbox := &storage.MockMailbox{} - ds.On("MailboxFor", "good").Return(goodbox, nil) msg1 := data1.MockMessage() msg2 := data2.MockMessage() - goodbox.On("GetMessages").Return([]storage.Message{msg1, msg2}, nil) + ds.AddMessage("good", msg1) + ds.AddMessage("good", msg2) // Check return code w, err = testRestGet(baseURL + "/mailbox/good") @@ -130,7 +102,7 @@ func TestRestMailboxList(t *testing.T) { t.Errorf("Failed to decode JSON: %v", err) } if len(result) != 2 { - t.Errorf("Expected 2 results, got %v", len(result)) + t.Fatalf("Expected 2 results, got %v", len(result)) } if errors := data1.CompareToJSONHeaderMap(result[0]); len(errors) > 0 { t.Logf("%v", result[0]) @@ -155,7 +127,7 @@ func TestRestMailboxList(t *testing.T) { func TestRestMessage(t *testing.T) { // Setup - ds := &storage.MockDataStore{} + ds := test.NewStore() logbuf := setupWebServer(ds) // Test invalid mailbox name @@ -169,10 +141,6 @@ func TestRestMessage(t *testing.T) { } // Test requesting a message that does not exist - emptybox := &storage.MockMailbox{} - ds.On("MailboxFor", "empty").Return(emptybox, nil) - emptybox.On("GetMessage", "0001").Return(&storage.MockMessage{}, storage.ErrNotExist) - w, err = testRestGet(baseURL + "/mailbox/empty/0001") expectCode = 404 if err != nil { @@ -182,9 +150,8 @@ func TestRestMessage(t *testing.T) { t.Errorf("Expected code %v, got %v", expectCode, w.Code) } - // Test MailboxFor error - ds.On("MailboxFor", "error").Return(&storage.MockMailbox{}, fmt.Errorf("Internal error")) - w, err = testRestGet(baseURL + "/mailbox/error/0001") + // Test GetMessage error + w, err = testRestGet(baseURL + "/mailbox/messageerr/0001") expectCode = 500 if err != nil { t.Fatal(err) @@ -200,20 +167,6 @@ func TestRestMessage(t *testing.T) { _, _ = io.Copy(os.Stderr, logbuf) } - // Test GetMessage error - error2box := &storage.MockMailbox{} - ds.On("MailboxFor", "error2").Return(error2box, nil) - error2box.On("GetMessage", "0001").Return(&storage.MockMessage{}, fmt.Errorf("Internal error 2")) - - w, err = testRestGet(baseURL + "/mailbox/error2/0001") - expectCode = 500 - if err != nil { - t.Fatal(err) - } - if w.Code != expectCode { - t.Errorf("Expected code %v, got %v", expectCode, w.Code) - } - // Test JSON message headers data1 := &InputMessageData{ Mailbox: "good", @@ -228,10 +181,8 @@ func TestRestMessage(t *testing.T) { Text: "This is some text", HTML: "This is some HTML", } - goodbox := &storage.MockMailbox{} - ds.On("MailboxFor", "good").Return(goodbox, nil) msg1 := data1.MockMessage() - goodbox.On("GetMessage", "0001").Return(msg1, nil) + ds.AddMessage("good", msg1) // Check return code w, err = testRestGet(baseURL + "/mailbox/good/0001") diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index 98cce65..4f6c4b6 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -513,12 +513,11 @@ func (ses *Session) sendMessageTop(msg storage.Message, lineCount int) { // Load the users mailbox func (ses *Session) loadMailbox() { - var err error - ses.messages, err = ses.mailbox.GetMessages() + m, err := ses.server.dataStore.GetMessages(ses.user) if err != nil { - ses.logError("Failed to load messages for %v", ses.user) + ses.logError("Failed to load messages for %v: %v", ses.user, err) } - + ses.messages = m ses.retainAll() } diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index a678d3a..3b0660d 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -478,10 +478,15 @@ func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) ss.logError("Error while closing message for %v: %v", r.mailbox, err) return false } + name, err := stringutil.ParseMailboxName(r.localPart) + if err != nil { + // This parse already succeeded when MailboxFor was called, shouldn't fail here. + return false + } // Broadcast message information broadcast := msghub.Message{ - Mailbox: r.mailbox.Name(), + Mailbox: name, ID: msg.ID(), From: msg.From(), To: msg.To(), diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index 5588cbb..0a03873 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -81,9 +81,36 @@ func DefaultStore() storage.Store { return New(cfg) } +// GetMessage returns the messages in the named mailbox, or an error. +func (fs *Store) GetMessage(mailbox, id string) (storage.Message, error) { + mb, err := fs.MailboxFor(mailbox) + if err != nil { + return nil, err + } + return mb.(*Mailbox).GetMessage(id) +} + +// GetMessages returns the messages in the named mailbox, or an error. +func (fs *Store) GetMessages(mailbox string) ([]storage.Message, error) { + mb, err := fs.MailboxFor(mailbox) + if err != nil { + return nil, err + } + return mb.(*Mailbox).GetMessages() +} + +// PurgeMessages deletes all messages in the named mailbox, or returns an error. +func (fs *Store) PurgeMessages(name string) error { + mb, err := fs.MailboxFor(name) + if err != nil { + return err + } + return mb.(*Mailbox).Purge() +} + // MailboxFor retrieves the Mailbox object for a specified email address, if the mailbox // does not exist, it will attempt to create it. -func (ds *Store) MailboxFor(emailAddress string) (storage.Mailbox, error) { +func (fs *Store) MailboxFor(emailAddress string) (storage.Mailbox, error) { name, err := stringutil.ParseMailboxName(emailAddress) if err != nil { return nil, err @@ -91,17 +118,17 @@ func (ds *Store) MailboxFor(emailAddress string) (storage.Mailbox, error) { dir := stringutil.HashMailboxName(name) s1 := dir[0:3] s2 := dir[0:6] - path := filepath.Join(ds.mailPath, s1, s2, dir) + path := filepath.Join(fs.mailPath, s1, s2, dir) indexPath := filepath.Join(path, indexFileName) - return &Mailbox{store: ds, name: name, dirName: dir, path: path, + return &Mailbox{store: fs, name: name, dirName: dir, path: path, indexPath: indexPath}, nil } // AllMailboxes returns a slice with all Mailboxes -func (ds *Store) AllMailboxes() ([]storage.Mailbox, error) { +func (fs *Store) AllMailboxes() ([]storage.Mailbox, error) { mailboxes := make([]storage.Mailbox, 0, 100) - infos1, err := ioutil.ReadDir(ds.mailPath) + infos1, err := ioutil.ReadDir(fs.mailPath) if err != nil { return nil, err } @@ -109,7 +136,7 @@ func (ds *Store) AllMailboxes() ([]storage.Mailbox, error) { for _, inf1 := range infos1 { if inf1.IsDir() { l1 := inf1.Name() - infos2, err := ioutil.ReadDir(filepath.Join(ds.mailPath, l1)) + infos2, err := ioutil.ReadDir(filepath.Join(fs.mailPath, l1)) if err != nil { return nil, err } @@ -117,7 +144,7 @@ func (ds *Store) AllMailboxes() ([]storage.Mailbox, error) { for _, inf2 := range infos2 { if inf2.IsDir() { l2 := inf2.Name() - infos3, err := ioutil.ReadDir(filepath.Join(ds.mailPath, l1, l2)) + infos3, err := ioutil.ReadDir(filepath.Join(fs.mailPath, l1, l2)) if err != nil { return nil, err } @@ -125,9 +152,9 @@ func (ds *Store) AllMailboxes() ([]storage.Mailbox, error) { for _, inf3 := range infos3 { if inf3.IsDir() { mbdir := inf3.Name() - mbpath := filepath.Join(ds.mailPath, l1, l2, mbdir) + mbpath := filepath.Join(fs.mailPath, l1, l2, mbdir) idx := filepath.Join(mbpath, indexFileName) - mb := &Mailbox{store: ds, dirName: mbdir, path: mbpath, + mb := &Mailbox{store: fs, dirName: mbdir, path: mbpath, indexPath: idx} mailboxes = append(mailboxes, mb) } @@ -141,13 +168,13 @@ func (ds *Store) AllMailboxes() ([]storage.Mailbox, error) { } // LockFor returns the RWMutex for this mailbox, or an error. -func (ds *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { +func (fs *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { name, err := stringutil.ParseMailboxName(emailAddress) if err != nil { return nil, err } hash := stringutil.HashMailboxName(name) - return ds.hashLock.Get(hash), nil + return fs.hashLock.Get(hash), nil } // Mailbox implements Mailbox, manages the mail for a specific user and diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index b02bdd1..26671c4 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -62,9 +62,7 @@ func TestFSDirStructure(t *testing.T) { assert.True(t, isFile(expect), "Expected %q to be a file", expect) // Delete message - mb, err := ds.MailboxFor(mbName) - assert.Nil(t, err) - msg, err := mb.GetMessage(id1) + msg, err := ds.GetMessage(mbName, id1) assert.Nil(t, err) err = msg.Delete() assert.Nil(t, err) @@ -76,7 +74,7 @@ func TestFSDirStructure(t *testing.T) { assert.True(t, isFile(expect), "Expected %q to be a file", expect) // Delete message - msg, err = mb.GetMessage(id2) + msg, err = ds.GetMessage(mbName, id2) assert.Nil(t, err) err = msg.Delete() assert.Nil(t, err) @@ -137,11 +135,7 @@ func TestFSDeliverMany(t *testing.T) { for i, subj := range subjects { // Check number of messages - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err := mb.GetMessages() + msgs, err := ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -151,11 +145,7 @@ func TestFSDeliverMany(t *testing.T) { deliverMessage(ds, mbName, subj, time.Now()) } - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err := mb.GetMessages() + msgs, err := ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -189,11 +179,7 @@ func TestFSDelete(t *testing.T) { deliverMessage(ds, mbName, subj, time.Now()) } - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err := mb.GetMessages() + msgs, err := ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -205,11 +191,7 @@ func TestFSDelete(t *testing.T) { _ = msgs[3].Delete() // Confirm deletion - mb, err = ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err = mb.GetMessages() + msgs, err = ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -225,11 +207,7 @@ func TestFSDelete(t *testing.T) { // Try appending one more deliverMessage(ds, mbName, "foxtrot", time.Now()) - mb, err = ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err = mb.GetMessages() + msgs, err = ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -263,11 +241,7 @@ func TestFSPurge(t *testing.T) { deliverMessage(ds, mbName, subj, time.Now()) } - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err := mb.GetMessages() + msgs, err := ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -275,15 +249,11 @@ func TestFSPurge(t *testing.T) { len(subjects), len(msgs)) // Purge mailbox - err = mb.Purge() + err = ds.PurgeMessages(mbName) assert.Nil(t, err) // Confirm deletion - mb, err = ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err = mb.GetMessages() + msgs, err = ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -315,12 +285,8 @@ func TestFSSize(t *testing.T) { sentSizes[i] = size } - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } for i, id := range sentIds { - msg, err := mb.GetMessage(id) + msg, err := ds.GetMessage(mbName, id) assert.Nil(t, err) expect := sentSizes[i] @@ -351,17 +317,12 @@ func TestFSMissing(t *testing.T) { sentIds[i] = id } - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - // Delete a message file without removing it from index - msg, err := mb.GetMessage(sentIds[1]) + msg, err := ds.GetMessage(mbName, sentIds[1]) assert.Nil(t, err) fmsg := msg.(*Message) _ = os.Remove(fmsg.rawPath()) - msg, err = mb.GetMessage(sentIds[1]) + msg, err = ds.GetMessage(mbName, sentIds[1]) assert.Nil(t, err) // Try to read parts of message @@ -392,11 +353,7 @@ func TestFSMessageCap(t *testing.T) { t.Logf("Delivered %q", subj) // Check number of messages - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err := mb.GetMessages() + msgs, err := ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -437,11 +394,7 @@ func TestFSNoMessageCap(t *testing.T) { t.Logf("Delivered %q", subj) // Check number of messages - mb, err := ds.MailboxFor(mbName) - if err != nil { - t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err) - } - msgs, err := mb.GetMessages() + msgs, err := ds.GetMessages(mbName) if err != nil { t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) } @@ -467,9 +420,7 @@ func TestGetLatestMessage(t *testing.T) { mbName := "james" // Test empty mailbox - mb, err := ds.MailboxFor(mbName) - assert.Nil(t, err) - msg, err := mb.GetMessage("latest") + msg, err := ds.GetMessage(mbName, "latest") assert.Nil(t, msg) assert.Error(t, err) @@ -480,23 +431,19 @@ func TestGetLatestMessage(t *testing.T) { id2, _ := deliverMessage(ds, mbName, "test 2", time.Now()) // Test get the latest message - mb, err = ds.MailboxFor(mbName) - assert.Nil(t, err) - msg, err = mb.GetMessage("latest") + msg, err = ds.GetMessage(mbName, "latest") assert.Nil(t, err) assert.True(t, msg.ID() == id2, "Expected %q to be equal to %q", msg.ID(), id2) // Deliver test message 3 id3, _ := deliverMessage(ds, mbName, "test 3", time.Now()) - mb, err = ds.MailboxFor(mbName) - assert.Nil(t, err) - msg, err = mb.GetMessage("latest") + msg, err = ds.GetMessage(mbName, "latest") assert.Nil(t, err) assert.True(t, msg.ID() == id3, "Expected %q to be equal to %q", msg.ID(), id3) // Test wrong id - _, err = mb.GetMessage("wrongid") + _, err = ds.GetMessage(mbName, "wrongid") assert.Error(t, err) if t.Failed() { diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index a137f9b..51620ed 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -21,6 +21,9 @@ var ( // Store is an interface to get Mailboxes stored in Inbucket type Store interface { + GetMessage(mailbox string, id string) (Message, error) + GetMessages(mailbox string) ([]Message, error) + PurgeMessages(mailbox string) error MailboxFor(emailAddress string) (Mailbox, error) AllMailboxes() ([]Mailbox, error) // LockFor is a temporary hack to fix #77 until Datastore revamp @@ -30,10 +33,7 @@ type Store interface { // Mailbox is an interface to get and manipulate messages in a DataStore type Mailbox interface { GetMessages() ([]Message, error) - GetMessage(id string) (Message, error) - Purge() error NewMessage() (Message, error) - Name() string String() string } diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index fc8dbab..8c51b3f 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -15,6 +15,24 @@ type MockDataStore struct { mock.Mock } +// GetMessage mock function +func (m *MockDataStore) GetMessage(name, id string) (Message, error) { + args := m.Called(name, id) + return args.Get(0).(Message), args.Error(1) +} + +// GetMessages mock function +func (m *MockDataStore) GetMessages(name string) ([]Message, error) { + args := m.Called(name) + return args.Get(0).([]Message), args.Error(1) +} + +// PurgeMessages mock function +func (m *MockDataStore) PurgeMessages(name string) error { + args := m.Called(name) + return args.Error(0) +} + // MailboxFor mock function func (m *MockDataStore) MailboxFor(name string) (Mailbox, error) { args := m.Called(name) @@ -61,12 +79,6 @@ func (m *MockMailbox) NewMessage() (Message, error) { return args.Get(0).(Message), args.Error(1) } -// Name mock function -func (m *MockMailbox) Name() string { - args := m.Called() - return args.String(0) -} - // String mock function func (m *MockMailbox) String() string { args := m.Called() diff --git a/pkg/test/storage.go b/pkg/test/storage.go new file mode 100644 index 0000000..29c24ce --- /dev/null +++ b/pkg/test/storage.go @@ -0,0 +1,47 @@ +package test + +import ( + "errors" + + "github.com/jhillyerd/inbucket/pkg/storage" +) + +// StoreStub stubs storage.Store for testing. +type StoreStub struct { + storage.Store + mailboxes map[string][]storage.Message +} + +// NewStore creates a new StoreStub. +func NewStore() *StoreStub { + return &StoreStub{ + mailboxes: make(map[string][]storage.Message), + } +} + +// AddMessage adds a message to the specified mailbox. +func (s *StoreStub) AddMessage(mailbox string, m storage.Message) { + msgs := s.mailboxes[mailbox] + s.mailboxes[mailbox] = append(msgs, m) +} + +// GetMessage gets a message by ID from the specified mailbox. +func (s *StoreStub) GetMessage(mailbox, id string) (storage.Message, error) { + if mailbox == "messageerr" { + return nil, errors.New("internal error") + } + for _, m := range s.mailboxes[mailbox] { + if m.ID() == id { + return m, nil + } + } + return nil, storage.ErrNotExist +} + +// GetMessages gets all the messages for the specified mailbox. +func (s *StoreStub) GetMessages(mailbox string) ([]storage.Message, error) { + if mailbox == "messageserr" { + return nil, errors.New("internal error") + } + return s.mailboxes[mailbox], nil +} diff --git a/pkg/webui/mailbox_controller.go b/pkg/webui/mailbox_controller.go index f9684b9..69b3a3f 100644 --- a/pkg/webui/mailbox_controller.go +++ b/pkg/webui/mailbox_controller.go @@ -72,12 +72,7 @@ func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - messages, err := mb.GetMessages() + messages, err := ctx.DataStore.GetMessages(name) if err != nil { // This doesn't indicate empty, likely an IO error return fmt.Errorf("Failed to get messages for %v: %v", name, err) @@ -99,12 +94,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - msg, err := mb.GetMessage(id) + msg, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -148,12 +138,7 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - message, err := mb.GetMessage(id) + message, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -172,8 +157,7 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er "ctx": ctx, "name": name, "message": message, - // TODO It is not really safe to render, need to sanitize, issue #5 - "body": template.HTML(mime.HTML), + "body": template.HTML(mime.HTML), }) } @@ -185,12 +169,7 @@ func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - message, err := mb.GetMessage(id) + message, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -231,12 +210,7 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Co http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - message, err := mb.GetMessage(id) + message, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -284,12 +258,7 @@ func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Contex http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - mb, err := ctx.DataStore.MailboxFor(name) - if err != nil { - // This doesn't indicate not found, likely an IO error - return fmt.Errorf("Failed to get mailbox for %q: %v", name, err) - } - message, err := mb.GetMessage(id) + message, err := ctx.DataStore.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil From d9b5e40c8764b4b04264584cf61d23940632d7e7 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 10 Mar 2018 22:05:10 -0800 Subject: [PATCH 03/26] storage: More refactoring for #69 - retention: Start from pkg main instead of server/smtp - file: Remove DefaultStore() constructor - storage: AllMailboxes replaced with VisitMailboxes for #69 - test: Stub VisitMailboxes for #80 --- cmd/inbucket/main.go | 9 ++++-- pkg/server/smtp/listener.go | 34 +++++++++------------ pkg/storage/file/fstore.go | 30 +++++++++---------- pkg/storage/file/fstore_test.go | 21 +++++++++---- pkg/storage/retention.go | 42 ++++++++++++-------------- pkg/storage/retention_test.go | 52 +++++++++++++-------------------- pkg/storage/storage.go | 2 +- pkg/storage/testing.go | 12 ++++---- pkg/test/storage.go | 11 +++++++ 9 files changed, 106 insertions(+), 107 deletions(-) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index 6e25e44..9f1d116 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -19,6 +19,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/server/pop3" "github.com/jhillyerd/inbucket/pkg/server/smtp" "github.com/jhillyerd/inbucket/pkg/server/web" + "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/storage/file" "github.com/jhillyerd/inbucket/pkg/webui" ) @@ -115,8 +116,11 @@ func main() { // Create message hub msgHub := msghub.New(rootCtx, config.GetWebConfig().MonitorHistory) - // Grab our datastore - ds := file.DefaultStore() + // Setup our datastore + dscfg := config.GetDataStoreConfig() + ds := file.New(dscfg) + retentionScanner := storage.NewRetentionScanner(dscfg, ds, shutdownChan) + retentionScanner.Start() // Start HTTP server web.Initialize(config.GetWebConfig(), shutdownChan, ds, msgHub) @@ -160,6 +164,7 @@ signalLoop: go timedExit() smtpServer.Drain() pop3Server.Drain() + retentionScanner.Join() removePIDFile() } diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index 4586174..83d5697 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -48,10 +48,9 @@ type Server struct { storeMessages bool // Dependencies - dataStore storage.Store // Mailbox/message store - globalShutdown chan bool // Shuts down Inbucket - msgHub *msghub.Hub // Pub/sub for message info - retentionScanner *storage.RetentionScanner // Deletes expired messages + dataStore storage.Store // Mailbox/message store + globalShutdown chan bool // Shuts down Inbucket + msgHub *msghub.Hub // Pub/sub for message info // State listener net.Listener // Incoming network connections @@ -86,18 +85,17 @@ func NewServer( ds storage.Store, msgHub *msghub.Hub) *Server { return &Server{ - host: fmt.Sprintf("%v:%v", cfg.IP4address, cfg.IP4port), - domain: cfg.Domain, - domainNoStore: strings.ToLower(cfg.DomainNoStore), - maxRecips: cfg.MaxRecipients, - maxIdleSeconds: cfg.MaxIdleSeconds, - maxMessageBytes: cfg.MaxMessageBytes, - storeMessages: cfg.StoreMessages, - globalShutdown: globalShutdown, - dataStore: ds, - msgHub: msgHub, - retentionScanner: storage.NewRetentionScanner(ds, globalShutdown), - waitgroup: new(sync.WaitGroup), + host: fmt.Sprintf("%v:%v", cfg.IP4address, cfg.IP4port), + domain: cfg.Domain, + domainNoStore: strings.ToLower(cfg.DomainNoStore), + maxRecips: cfg.MaxRecipients, + maxIdleSeconds: cfg.MaxIdleSeconds, + maxMessageBytes: cfg.MaxMessageBytes, + storeMessages: cfg.StoreMessages, + globalShutdown: globalShutdown, + dataStore: ds, + msgHub: msgHub, + waitgroup: new(sync.WaitGroup), } } @@ -124,9 +122,6 @@ func (s *Server) Start(ctx context.Context) { log.Infof("Messages sent to domain '%v' will be discarded", s.domainNoStore) } - // Start retention scanner - s.retentionScanner.Start() - // Listener go routine go s.serve(ctx) @@ -195,5 +190,4 @@ func (s *Server) Drain() { // Wait for sessions to close s.waitgroup.Wait() log.Tracef("SMTP connections have drained") - s.retentionScanner.Join() } diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index 0a03873..ddf6d1c 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -74,13 +74,6 @@ func New(cfg config.DataStoreConfig) storage.Store { return &Store{path: path, mailPath: mailPath, messageCap: cfg.MailboxMsgCap} } -// DefaultStore creates a new DataStore object. It uses the inbucket.Config object to -// construct it's path. -func DefaultStore() storage.Store { - cfg := config.GetDataStoreConfig() - return New(cfg) -} - // GetMessage returns the messages in the named mailbox, or an error. func (fs *Store) GetMessage(mailbox, id string) (storage.Message, error) { mb, err := fs.MailboxFor(mailbox) @@ -125,12 +118,12 @@ func (fs *Store) MailboxFor(emailAddress string) (storage.Mailbox, error) { indexPath: indexPath}, nil } -// AllMailboxes returns a slice with all Mailboxes -func (fs *Store) AllMailboxes() ([]storage.Mailbox, error) { - mailboxes := make([]storage.Mailbox, 0, 100) +// VisitMailboxes accepts a function that will be called with the messages in each mailbox while it +// continues to return true. +func (fs *Store) VisitMailboxes(f func([]storage.Message) (cont bool)) error { infos1, err := ioutil.ReadDir(fs.mailPath) if err != nil { - return nil, err + return err } // Loop over level 1 directories for _, inf1 := range infos1 { @@ -138,7 +131,7 @@ func (fs *Store) AllMailboxes() ([]storage.Mailbox, error) { l1 := inf1.Name() infos2, err := ioutil.ReadDir(filepath.Join(fs.mailPath, l1)) if err != nil { - return nil, err + return err } // Loop over level 2 directories for _, inf2 := range infos2 { @@ -146,7 +139,7 @@ func (fs *Store) AllMailboxes() ([]storage.Mailbox, error) { l2 := inf2.Name() infos3, err := ioutil.ReadDir(filepath.Join(fs.mailPath, l1, l2)) if err != nil { - return nil, err + return err } // Loop over mailboxes for _, inf3 := range infos3 { @@ -156,15 +149,20 @@ func (fs *Store) AllMailboxes() ([]storage.Mailbox, error) { idx := filepath.Join(mbpath, indexFileName) mb := &Mailbox{store: fs, dirName: mbdir, path: mbpath, indexPath: idx} - mailboxes = append(mailboxes, mb) + msgs, err := mb.GetMessages() + if err != nil { + return err + } + if !f(msgs) { + return nil + } } } } } } } - - return mailboxes, nil + return nil } // LockFor returns the RWMutex for this mailbox, or an error. diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 26671c4..8247de9 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/storage" "github.com/stretchr/testify/assert" ) @@ -97,12 +98,12 @@ func TestFSDirStructure(t *testing.T) { } } -// Test FileDataStore.AllMailboxes() -func TestFSAllMailboxes(t *testing.T) { +// TestFSVisitMailboxes tests VisitMailboxes +func TestFSVisitMailboxes(t *testing.T) { ds, logbuf := setupDataStore(config.DataStoreConfig{}) defer teardownDataStore(ds) - - for _, name := range []string{"abby", "bill", "christa", "donald", "evelyn"} { + boxes := []string{"abby", "bill", "christa", "donald", "evelyn"} + for _, name := range boxes { // Create day old message date := time.Now().Add(-24 * time.Hour) deliverMessage(ds, name, "Old Message", date) @@ -112,9 +113,17 @@ func TestFSAllMailboxes(t *testing.T) { deliverMessage(ds, name, "New Message", date) } - mboxes, err := ds.AllMailboxes() + seen := 0 + err := ds.VisitMailboxes(func(messages []storage.Message) bool { + seen++ + count := len(messages) + if count != 2 { + t.Errorf("got: %v messages, want: 2", count) + } + return true + }) assert.Nil(t, err) - assert.Equal(t, len(mboxes), 5) + assert.Equal(t, 5, seen) if t.Failed() { // Wait for handler to finish logging diff --git a/pkg/storage/retention.go b/pkg/storage/retention.go index f067843..6269443 100644 --- a/pkg/storage/retention.go +++ b/pkg/storage/retention.go @@ -52,10 +52,12 @@ type RetentionScanner struct { retentionSleep time.Duration } -// NewRetentionScanner launches a go-routine that scans for expired -// messages, following the configured interval -func NewRetentionScanner(ds Store, shutdownChannel chan bool) *RetentionScanner { - cfg := config.GetDataStoreConfig() +// NewRetentionScanner configures a new RententionScanner. +func NewRetentionScanner( + cfg config.DataStoreConfig, + ds Store, + shutdownChannel chan bool, +) *RetentionScanner { rs := &RetentionScanner{ globalShutdown: shutdownChannel, retentionShutdown: make(chan bool), @@ -97,7 +99,7 @@ retentionLoop: } // Kickoff scan start = time.Now() - if err := rs.doScan(); err != nil { + if err := rs.DoScan(); err != nil { log.Errorf("Error during retention scan: %v", err) } // Check for global shutdown @@ -111,28 +113,17 @@ retentionLoop: close(rs.retentionShutdown) } -// doScan does a single pass of all mailboxes looking for messages that can be purged -func (rs *RetentionScanner) doScan() error { +// DoScan does a single pass of all mailboxes looking for messages that can be purged. +func (rs *RetentionScanner) DoScan() error { log.Tracef("Starting retention scan") cutoff := time.Now().Add(-1 * rs.retentionPeriod) - mboxes, err := rs.ds.AllMailboxes() - if err != nil { - return err - } retained := 0 - // Loop over all mailboxes - for _, mb := range mboxes { - messages, err := mb.GetMessages() - if err != nil { - return err - } - // Loop over all messages in mailbox + // Loop over all mailboxes. + err := rs.ds.VisitMailboxes(func(messages []Message) bool { for _, msg := range messages { if msg.Date().Before(cutoff) { log.Tracef("Purging expired message %v", msg.ID()) - err = msg.Delete() - if err != nil { - // Log but don't abort + if err := msg.Delete(); err != nil { log.Errorf("Failed to purge message %v: %v", msg.ID(), err) } else { expRetentionDeletesTotal.Add(1) @@ -141,14 +132,17 @@ func (rs *RetentionScanner) doScan() error { retained++ } } - // Sleep after completing a mailbox select { case <-rs.globalShutdown: log.Tracef("Retention scan aborted due to shutdown") - return nil + return false case <-time.After(rs.retentionSleep): // Reduce disk thrashing } + return true + }) + if err != nil { + return err } // Update metrics setRetentionScanCompleted(time.Now()) @@ -156,7 +150,7 @@ func (rs *RetentionScanner) doScan() error { return nil } -// Join does not retun until the retention scanner has shut down +// Join does not return until the retention scanner has shut down. func (rs *RetentionScanner) Join() { if rs.retentionShutdown != nil { <-rs.retentionShutdown diff --git a/pkg/storage/retention_test.go b/pkg/storage/retention_test.go index ae221e9..bd862cf 100644 --- a/pkg/storage/retention_test.go +++ b/pkg/storage/retention_test.go @@ -1,19 +1,17 @@ -package storage +package storage_test import ( "fmt" "testing" "time" + + "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/test" ) func TestDoRetentionScan(t *testing.T) { - // Create mock objects - mds := &MockDataStore{} - - mb1 := &MockMailbox{} - mb2 := &MockMailbox{} - mb3 := &MockMailbox{} - + ds := test.NewStore() // Mockup some different aged messages (num is in hours) new1 := mockMessage(0) new2 := mockMessage(1) @@ -21,36 +19,26 @@ func TestDoRetentionScan(t *testing.T) { old1 := mockMessage(4) old2 := mockMessage(12) old3 := mockMessage(24) - - // First it should ask for all mailboxes - mds.On("AllMailboxes").Return([]Mailbox{mb1, mb2, mb3}, nil) - - // Then for all messages on each box - mb1.On("GetMessages").Return([]Message{new1, old1, old2}, nil) - mb2.On("GetMessages").Return([]Message{old3, new2}, nil) - mb3.On("GetMessages").Return([]Message{new3}, nil) - + ds.AddMessage("mb1", new1) + ds.AddMessage("mb1", old1) + ds.AddMessage("mb1", old2) + ds.AddMessage("mb2", old3) + ds.AddMessage("mb2", new2) + ds.AddMessage("mb3", new3) // Test 4 hour retention - rs := &RetentionScanner{ - ds: mds, - retentionPeriod: 4*time.Hour - time.Minute, - retentionSleep: 0, + cfg := config.DataStoreConfig{ + RetentionMinutes: 239, + RetentionSleep: 0, } - if err := rs.doScan(); err != nil { + shutdownChan := make(chan bool) + rs := storage.NewRetentionScanner(cfg, ds, shutdownChan) + if err := rs.DoScan(); err != nil { t.Error(err) } - - // Check our assertions - mds.AssertExpectations(t) - mb1.AssertExpectations(t) - mb2.AssertExpectations(t) - mb3.AssertExpectations(t) - // Delete should not have been called on new messages new1.AssertNotCalled(t, "Delete") new2.AssertNotCalled(t, "Delete") new3.AssertNotCalled(t, "Delete") - // Delete should have been called once on old messages old1.AssertNumberOfCalls(t, "Delete", 1) old2.AssertNumberOfCalls(t, "Delete", 1) @@ -58,8 +46,8 @@ func TestDoRetentionScan(t *testing.T) { } // Make a MockMessage of a specific age -func mockMessage(ageHours int) *MockMessage { - msg := &MockMessage{} +func mockMessage(ageHours int) *storage.MockMessage { + msg := &storage.MockMessage{} msg.On("ID").Return(fmt.Sprintf("MSG[age=%vh]", ageHours)) msg.On("Date").Return(time.Now().Add(time.Duration(ageHours*-1) * time.Hour)) msg.On("Delete").Return(nil) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 51620ed..b18bf33 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -24,8 +24,8 @@ type Store interface { GetMessage(mailbox string, id string) (Message, error) GetMessages(mailbox string) ([]Message, error) PurgeMessages(mailbox string) error + VisitMailboxes(f func([]Message) (cont bool)) error MailboxFor(emailAddress string) (Mailbox, error) - AllMailboxes() ([]Mailbox, error) // LockFor is a temporary hack to fix #77 until Datastore revamp LockFor(emailAddress string) (*sync.RWMutex, error) } diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index 8c51b3f..64d7bf0 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -39,17 +39,17 @@ func (m *MockDataStore) MailboxFor(name string) (Mailbox, error) { return args.Get(0).(Mailbox), args.Error(1) } -// AllMailboxes mock function -func (m *MockDataStore) AllMailboxes() ([]Mailbox, error) { - args := m.Called() - return args.Get(0).([]Mailbox), args.Error(1) -} - // LockFor mock function returns a new RWMutex, never errors. func (m *MockDataStore) LockFor(name string) (*sync.RWMutex, error) { return &sync.RWMutex{}, nil } +// VisitMailboxes accepts a function that will be called with the messages in each mailbox while it +// continues to return true. +func (m *MockDataStore) VisitMailboxes(f func([]Message) (cont bool)) error { + return nil +} + // MockMailbox is a shared mock for unit testing type MockMailbox struct { mock.Mock diff --git a/pkg/test/storage.go b/pkg/test/storage.go index 29c24ce..3f0fcbd 100644 --- a/pkg/test/storage.go +++ b/pkg/test/storage.go @@ -45,3 +45,14 @@ func (s *StoreStub) GetMessages(mailbox string) ([]storage.Message, error) { } return s.mailboxes[mailbox], nil } + +// VisitMailboxes accepts a function that will be called with the messages in each mailbox while it +// continues to return true. +func (s *StoreStub) VisitMailboxes(f func([]storage.Message) (cont bool)) error { + for _, v := range s.mailboxes { + if !f(v) { + return nil + } + } + return nil +} From 137466f89b4cc6f0cac1d46ec9b80946f5117ca2 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 11 Mar 2018 10:48:50 -0700 Subject: [PATCH 04/26] storage: Move NewMessage() into Store interface for #69 --- pkg/server/smtp/handler.go | 2 +- pkg/server/smtp/handler_test.go | 4 ++-- pkg/storage/file/fstore.go | 9 +++++++++ pkg/storage/file/fstore_test.go | 6 +----- pkg/storage/storage.go | 3 ++- pkg/storage/testing.go | 6 ++++++ pkg/test/storage.go | 5 +++++ 7 files changed, 26 insertions(+), 9 deletions(-) diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 3b0660d..228718a 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -451,7 +451,7 @@ func (ss *Session) dataHandler() { // deliverMessage creates and populates a new Message for the specified recipient func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) { - msg, err := r.mailbox.NewMessage() + msg, err := ss.server.dataStore.NewMessage(r.localPart) if err != nil { ss.logError("Failed to create message for %q: %s", r.localPart, err) return false diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index b53b651..b8603a5 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -148,7 +148,7 @@ func TestMailState(t *testing.T) { mb1 := &storage.MockMailbox{} msg1 := &storage.MockMessage{} mds.On("MailboxFor", "u1").Return(mb1, nil) - mb1.On("NewMessage").Return(msg1, nil) + mds.On("NewMessage", "u1").Return(msg1, nil) mb1.On("Name").Return("u1") msg1.On("ID").Return("") msg1.On("From").Return("") @@ -263,7 +263,7 @@ func TestDataState(t *testing.T) { mb1 := &storage.MockMailbox{} msg1 := &storage.MockMessage{} mds.On("MailboxFor", "u1").Return(mb1, nil) - mb1.On("NewMessage").Return(msg1, nil) + mds.On("NewMessage", "u1").Return(msg1, nil) mb1.On("Name").Return("u1") msg1.On("ID").Return("") msg1.On("From").Return("") diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index ddf6d1c..b3a2e5e 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -175,6 +175,15 @@ func (fs *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { return fs.hashLock.Get(hash), nil } +// NewMessage is temproary until #69 MessageData refactor +func (fs *Store) NewMessage(mailbox string) (storage.Message, error) { + mb, err := fs.MailboxFor(mailbox) + if err != nil { + return nil, err + } + return mb.(*Mailbox).NewMessage() +} + // Mailbox implements Mailbox, manages the mail for a specific user and // correlates to a particular directory on disk. type Mailbox struct { diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 8247de9..a7bb039 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -490,13 +490,9 @@ func deliverMessage(ds *Store, mbName string, subject string, testMsg = append(testMsg, []byte("\r\n")...) testMsg = append(testMsg, []byte("Test Body\r\n")...) - mb, err := ds.MailboxFor(mbName) - if err != nil { - panic(err) - } // Create message object id = generateID(date) - msg, err := mb.NewMessage() + msg, err := ds.NewMessage(mbName) if err != nil { panic(err) } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index b18bf33..54adb58 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -28,12 +28,13 @@ type Store interface { MailboxFor(emailAddress string) (Mailbox, error) // LockFor is a temporary hack to fix #77 until Datastore revamp LockFor(emailAddress string) (*sync.RWMutex, error) + // NewMessage is temproary until #69 MessageData refactor + NewMessage(mailbox string) (Message, error) } // Mailbox is an interface to get and manipulate messages in a DataStore type Mailbox interface { GetMessages() ([]Message, error) - NewMessage() (Message, error) String() string } diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index 64d7bf0..8757230 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -44,6 +44,12 @@ func (m *MockDataStore) LockFor(name string) (*sync.RWMutex, error) { return &sync.RWMutex{}, nil } +// NewMessage temporary for #69 +func (m *MockDataStore) NewMessage(mailbox string) (Message, error) { + args := m.Called(mailbox) + return args.Get(0).(Message), args.Error(1) +} + // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it // continues to return true. func (m *MockDataStore) VisitMailboxes(f func([]Message) (cont bool)) error { diff --git a/pkg/test/storage.go b/pkg/test/storage.go index 3f0fcbd..fab78ca 100644 --- a/pkg/test/storage.go +++ b/pkg/test/storage.go @@ -56,3 +56,8 @@ func (s *StoreStub) VisitMailboxes(f func([]storage.Message) (cont bool)) error } return nil } + +// NewMessage is temproary until #69 MessageData refactor +func (s *StoreStub) NewMessage(mailbox string) (storage.Message, error) { + return nil, nil +} From 12ad0cb3f0abec74c191e3ab62293792b1062cae Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 11 Mar 2018 11:54:35 -0700 Subject: [PATCH 05/26] storage: Eliminate storage.Mailbox interface for #69 storage/file Mailbox has been renamed mbox, and is now just an implementation detail. --- pkg/server/pop3/handler.go | 17 ------ pkg/server/smtp/handler.go | 15 ++---- pkg/server/smtp/handler_test.go | 6 --- pkg/storage/file/fmessage.go | 8 ++- pkg/storage/file/fstore.go | 94 ++++++++++++++------------------- pkg/storage/storage.go | 7 --- pkg/storage/testing.go | 41 -------------- 7 files changed, 45 insertions(+), 143 deletions(-) diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index 4f6c4b6..5e3c665 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -65,7 +65,6 @@ type Session struct { state State // Current session state reader *bufio.Reader // Buffered reader for our net conn user string // Mailbox name - mailbox storage.Mailbox // Mailbox instance messages []storage.Message // Slice of messages in mailbox retain []bool // Messages to retain upon UPDATE (true=retain) msgCount int // Number of undeleted messages @@ -195,14 +194,6 @@ func (ses *Session) authorizationHandler(cmd string, args []string) { if ses.user == "" { ses.ooSeq(cmd) } else { - var err error - ses.mailbox, err = ses.server.dataStore.MailboxFor(ses.user) - if err != nil { - ses.logError("Failed to open mailbox for %v", ses.user) - ses.send(fmt.Sprintf("-ERR Failed to open mailbox for %v", ses.user)) - ses.enterState(QUIT) - return - } ses.loadMailbox() ses.send(fmt.Sprintf("+OK Found %v messages for %v", ses.msgCount, ses.user)) ses.enterState(TRANSACTION) @@ -214,14 +205,6 @@ func (ses *Session) authorizationHandler(cmd string, args []string) { return } ses.user = args[0] - var err error - ses.mailbox, err = ses.server.dataStore.MailboxFor(ses.user) - if err != nil { - ses.logError("Failed to open mailbox for %v", ses.user) - ses.send(fmt.Sprintf("-ERR Failed to open mailbox for %v", ses.user)) - ses.enterState(QUIT) - return - } ses.loadMailbox() ses.send(fmt.Sprintf("+OK Found %v messages for %v", ses.msgCount, ses.user)) ses.enterState(TRANSACTION) diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 228718a..50bf9ef 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -14,7 +14,6 @@ import ( "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/msghub" - "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/stringutil" ) @@ -73,7 +72,6 @@ var commands = map[string]bool{ // recipientDetails for message delivery type recipientDetails struct { address, localPart, domainPart string - mailbox storage.Mailbox } // Session holds the state of an SMTP session @@ -365,14 +363,7 @@ func (ss *Session) dataHandler() { } if strings.ToLower(domain) != ss.server.domainNoStore { // Not our "no store" domain, so store the message - mb, err := ss.server.dataStore.MailboxFor(local) - if err != nil { - ss.logError("Failed to open mailbox for %q: %s", local, err) - ss.send(fmt.Sprintf("451 Failed to open mailbox for %v", local)) - ss.reset() - return - } - recipients = append(recipients, recipientDetails{recip, local, domain, mb}) + recipients = append(recipients, recipientDetails{recip, local, domain}) } else { log.Tracef("Not storing message for %q", recip) } @@ -469,13 +460,13 @@ func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) // Append lines from msgBuf for _, line := range msgBuf { if err := msg.Append(line); err != nil { - ss.logError("Failed to append to mailbox %v: %v", r.mailbox, err) + ss.logError("Failed to append to mailbox %v: %v", r.localPart, err) // Should really cleanup the crap on filesystem return false } } if err := msg.Close(); err != nil { - ss.logError("Error while closing message for %v: %v", r.mailbox, err) + ss.logError("Error while closing message for %v: %v", r.localPart, err) return false } name, err := stringutil.ParseMailboxName(r.localPart) diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index b8603a5..29098ac 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -145,11 +145,8 @@ func TestReadyState(t *testing.T) { func TestMailState(t *testing.T) { // Setup mock objects mds := &storage.MockDataStore{} - mb1 := &storage.MockMailbox{} msg1 := &storage.MockMessage{} - mds.On("MailboxFor", "u1").Return(mb1, nil) mds.On("NewMessage", "u1").Return(msg1, nil) - mb1.On("Name").Return("u1") msg1.On("ID").Return("") msg1.On("From").Return("") msg1.On("To").Return(make([]string, 0)) @@ -260,11 +257,8 @@ func TestMailState(t *testing.T) { func TestDataState(t *testing.T) { // Setup mock objects mds := &storage.MockDataStore{} - mb1 := &storage.MockMailbox{} msg1 := &storage.MockMessage{} - mds.On("MailboxFor", "u1").Return(mb1, nil) mds.On("NewMessage", "u1").Return(msg1, nil) - mb1.On("Name").Return("u1") msg1.On("ID").Return("") msg1.On("From").Return("") msg1.On("To").Return(make([]string, 0)) diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index 4e496f8..b325678 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -18,7 +18,7 @@ import ( // Message implements Message and contains a little bit of data about a // particular email message, and methods to retrieve the rest of it from disk. type Message struct { - mailbox *Mailbox + mailbox *mbox // Stored in GOB Fid string Fdate time.Time @@ -32,16 +32,15 @@ type Message struct { writer *bufio.Writer } -// NewMessage creates a new FileMessage object and sets the Date and Id fields. +// newMessage creates a new FileMessage object and sets the Date and ID fields. // It will also delete messages over messageCap if configured. -func (mb *Mailbox) NewMessage() (storage.Message, error) { +func (mb *mbox) newMessage() (storage.Message, error) { // Load index if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err } } - // Delete old messages over messageCap if mb.store.messageCap > 0 { for len(mb.messages) >= mb.store.messageCap { @@ -51,7 +50,6 @@ func (mb *Mailbox) NewMessage() (storage.Message, error) { } } } - date := time.Now() id := generateID(date) return &Message{mailbox: mb, Fid: id, Fdate: date, writable: true}, nil diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index b3a2e5e..fb09ad0 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -76,46 +76,29 @@ func New(cfg config.DataStoreConfig) storage.Store { // GetMessage returns the messages in the named mailbox, or an error. func (fs *Store) GetMessage(mailbox, id string) (storage.Message, error) { - mb, err := fs.MailboxFor(mailbox) + mb, err := fs.mbox(mailbox) if err != nil { return nil, err } - return mb.(*Mailbox).GetMessage(id) + return mb.getMessage(id) } // GetMessages returns the messages in the named mailbox, or an error. func (fs *Store) GetMessages(mailbox string) ([]storage.Message, error) { - mb, err := fs.MailboxFor(mailbox) + mb, err := fs.mbox(mailbox) if err != nil { return nil, err } - return mb.(*Mailbox).GetMessages() + return mb.getMessages() } // PurgeMessages deletes all messages in the named mailbox, or returns an error. -func (fs *Store) PurgeMessages(name string) error { - mb, err := fs.MailboxFor(name) +func (fs *Store) PurgeMessages(mailbox string) error { + mb, err := fs.mbox(mailbox) if err != nil { return err } - return mb.(*Mailbox).Purge() -} - -// MailboxFor retrieves the Mailbox object for a specified email address, if the mailbox -// does not exist, it will attempt to create it. -func (fs *Store) MailboxFor(emailAddress string) (storage.Mailbox, error) { - name, err := stringutil.ParseMailboxName(emailAddress) - if err != nil { - return nil, err - } - dir := stringutil.HashMailboxName(name) - s1 := dir[0:3] - s2 := dir[0:6] - path := filepath.Join(fs.mailPath, s1, s2, dir) - indexPath := filepath.Join(path, indexFileName) - - return &Mailbox{store: fs, name: name, dirName: dir, path: path, - indexPath: indexPath}, nil + return mb.purge() } // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it @@ -147,9 +130,9 @@ func (fs *Store) VisitMailboxes(f func([]storage.Message) (cont bool)) error { mbdir := inf3.Name() mbpath := filepath.Join(fs.mailPath, l1, l2, mbdir) idx := filepath.Join(mbpath, indexFileName) - mb := &Mailbox{store: fs, dirName: mbdir, path: mbpath, + mb := &mbox{store: fs, dirName: mbdir, path: mbpath, indexPath: idx} - msgs, err := mb.GetMessages() + msgs, err := mb.getMessages() if err != nil { return err } @@ -177,16 +160,31 @@ func (fs *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { // NewMessage is temproary until #69 MessageData refactor func (fs *Store) NewMessage(mailbox string) (storage.Message, error) { - mb, err := fs.MailboxFor(mailbox) + mb, err := fs.mbox(mailbox) if err != nil { return nil, err } - return mb.(*Mailbox).NewMessage() + return mb.newMessage() } -// Mailbox implements Mailbox, manages the mail for a specific user and -// correlates to a particular directory on disk. -type Mailbox struct { +// mbox returns the named mailbox. +func (fs *Store) mbox(mailbox string) (*mbox, error) { + name, err := stringutil.ParseMailboxName(mailbox) + if err != nil { + return nil, err + } + dir := stringutil.HashMailboxName(name) + s1 := dir[0:3] + s2 := dir[0:6] + path := filepath.Join(fs.mailPath, s1, s2, dir) + indexPath := filepath.Join(path, indexFileName) + + return &mbox{store: fs, name: name, dirName: dir, path: path, + indexPath: indexPath}, nil +} + +// mbox manages the mail for a specific user and correlates to a particular directory on disk. +type mbox struct { store *Store name string dirName string @@ -196,25 +194,14 @@ type Mailbox struct { messages []*Message } -// Name of the mailbox -func (mb *Mailbox) Name() string { - return mb.name -} - -// String renders the name and directory path of the mailbox -func (mb *Mailbox) String() string { - return mb.name + "[" + mb.dirName + "]" -} - -// GetMessages scans the mailbox directory for .gob files and decodes them into +// getMessages scans the mailbox directory for .gob files and decodes them into // a slice of Message objects. -func (mb *Mailbox) GetMessages() ([]storage.Message, error) { +func (mb *mbox) getMessages() ([]storage.Message, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err } } - messages := make([]storage.Message, len(mb.messages)) for i, m := range mb.messages { messages[i] = m @@ -222,35 +209,32 @@ func (mb *Mailbox) GetMessages() ([]storage.Message, error) { return messages, nil } -// GetMessage decodes a single message by Id and returns a Message object -func (mb *Mailbox) GetMessage(id string) (storage.Message, error) { +// getMessage decodes a single message by ID and returns a Message object. +func (mb *mbox) getMessage(id string) (storage.Message, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err } } - if id == "latest" && len(mb.messages) != 0 { return mb.messages[len(mb.messages)-1], nil } - for _, m := range mb.messages { if m.Fid == id { return m, nil } } - return nil, storage.ErrNotExist } -// Purge deletes all messages in this mailbox -func (mb *Mailbox) Purge() error { +// purge deletes all messages in this mailbox. +func (mb *mbox) purge() error { mb.messages = mb.messages[:0] return mb.writeIndex() } // readIndex loads the mailbox index data from disk -func (mb *Mailbox) readIndex() error { +func (mb *mbox) readIndex() error { // Clear message slice, open index mb.messages = mb.messages[:0] // Lock for reading @@ -293,7 +277,7 @@ func (mb *Mailbox) readIndex() error { } // writeIndex overwrites the index on disk with the current mailbox data -func (mb *Mailbox) writeIndex() error { +func (mb *mbox) writeIndex() error { // Lock for writing indexMx.Lock() defer indexMx.Unlock() @@ -335,7 +319,7 @@ func (mb *Mailbox) writeIndex() error { } // createDir checks for the presence of the path for this mailbox, creates it if needed -func (mb *Mailbox) createDir() error { +func (mb *mbox) createDir() error { dirMx.Lock() defer dirMx.Unlock() if _, err := os.Stat(mb.path); err != nil { @@ -348,7 +332,7 @@ func (mb *Mailbox) createDir() error { } // removeDir removes the mailbox, plus empty higher level directories -func (mb *Mailbox) removeDir() error { +func (mb *mbox) removeDir() error { dirMx.Lock() defer dirMx.Unlock() // remove mailbox dir, including index file diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 54adb58..83cf635 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -25,19 +25,12 @@ type Store interface { GetMessages(mailbox string) ([]Message, error) PurgeMessages(mailbox string) error VisitMailboxes(f func([]Message) (cont bool)) error - MailboxFor(emailAddress string) (Mailbox, error) // LockFor is a temporary hack to fix #77 until Datastore revamp LockFor(emailAddress string) (*sync.RWMutex, error) // NewMessage is temproary until #69 MessageData refactor NewMessage(mailbox string) (Message, error) } -// Mailbox is an interface to get and manipulate messages in a DataStore -type Mailbox interface { - GetMessages() ([]Message, error) - String() string -} - // Message is an interface for a single message in a Mailbox type Message interface { ID() string diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index 8757230..6b2604d 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -33,12 +33,6 @@ func (m *MockDataStore) PurgeMessages(name string) error { return args.Error(0) } -// MailboxFor mock function -func (m *MockDataStore) MailboxFor(name string) (Mailbox, error) { - args := m.Called(name) - return args.Get(0).(Mailbox), args.Error(1) -} - // LockFor mock function returns a new RWMutex, never errors. func (m *MockDataStore) LockFor(name string) (*sync.RWMutex, error) { return &sync.RWMutex{}, nil @@ -56,41 +50,6 @@ func (m *MockDataStore) VisitMailboxes(f func([]Message) (cont bool)) error { return nil } -// MockMailbox is a shared mock for unit testing -type MockMailbox struct { - mock.Mock -} - -// GetMessages mock function -func (m *MockMailbox) GetMessages() ([]Message, error) { - args := m.Called() - return args.Get(0).([]Message), args.Error(1) -} - -// GetMessage mock function -func (m *MockMailbox) GetMessage(id string) (Message, error) { - args := m.Called(id) - return args.Get(0).(Message), args.Error(1) -} - -// Purge mock function -func (m *MockMailbox) Purge() error { - args := m.Called() - return args.Error(0) -} - -// NewMessage mock function -func (m *MockMailbox) NewMessage() (Message, error) { - args := m.Called() - return args.Get(0).(Message), args.Error(1) -} - -// String mock function -func (m *MockMailbox) String() string { - args := m.Called() - return args.String(0) -} - // MockMessage is a shared mock for unit testing type MockMessage struct { mock.Mock From 487e491d6f0c32bf493f2b931ea56bebdf51b99f Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 11 Mar 2018 15:01:40 -0700 Subject: [PATCH 06/26] storage: Message refactoring for #69 - Message interface renamed to StoreMessage - Message.Delete becomes Store.RemoveMessage - Added deleted message tracking to Store stub for #80 --- pkg/rest/apiv1_controller.go | 8 +--- pkg/server/pop3/handler.go | 28 ++++++------- pkg/server/smtp/handler_test.go | 13 +++--- pkg/storage/file/fmessage.go | 35 ++++------------ pkg/storage/file/fstore.go | 74 ++++++++++++++++++++++++++------- pkg/storage/file/fstore_test.go | 20 +++++---- pkg/storage/retention.go | 6 +-- pkg/storage/retention_test.go | 22 +++++++--- pkg/storage/storage.go | 21 +++++----- pkg/storage/testing.go | 32 ++++++++------ pkg/test/storage.go | 49 ++++++++++++++++++---- 11 files changed, 190 insertions(+), 118 deletions(-) diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index 9d7d1c6..6424648 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -158,18 +158,14 @@ func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return err } - message, err := ctx.DataStore.GetMessage(name, id) + err = ctx.DataStore.RemoveMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil } if err != nil { // This doesn't indicate missing, likely an IO error - return fmt.Errorf("GetMessage(%q) failed: %v", id, err) - } - err = message.Delete() - if err != nil { - return fmt.Errorf("Delete(%q) failed: %v", id, err) + return fmt.Errorf("RemoveMessage(%q) failed: %v", id, err) } return web.RenderJSON(w, "OK") diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index 5e3c665..c022619 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -57,17 +57,17 @@ var commands = map[string]bool{ // Session defines an active POP3 session type Session struct { - server *Server // Reference to the server we belong to - id int // Session ID number - conn net.Conn // Our network connection - remoteHost string // IP address of client - sendError error // Used to bail out of read loop on send error - state State // Current session state - reader *bufio.Reader // Buffered reader for our net conn - user string // Mailbox name - messages []storage.Message // Slice of messages in mailbox - retain []bool // Messages to retain upon UPDATE (true=retain) - msgCount int // Number of undeleted messages + server *Server // Reference to the server we belong to + id int // Session ID number + conn net.Conn // Our network connection + remoteHost string // IP address of client + sendError error // Used to bail out of read loop on send error + state State // Current session state + reader *bufio.Reader // Buffered reader for our net conn + user string // Mailbox name + messages []storage.StoreMessage // Slice of messages in mailbox + retain []bool // Messages to retain upon UPDATE (true=retain) + msgCount int // Number of undeleted messages } // NewSession creates a new POP3 session @@ -415,7 +415,7 @@ func (ses *Session) transactionHandler(cmd string, args []string) { } // Send the contents of the message to the client -func (ses *Session) sendMessage(msg storage.Message) { +func (ses *Session) sendMessage(msg storage.StoreMessage) { reader, err := msg.RawReader() if err != nil { ses.logError("Failed to read message for RETR command") @@ -448,7 +448,7 @@ func (ses *Session) sendMessage(msg storage.Message) { } // Send the headers plus the top N lines to the client -func (ses *Session) sendMessageTop(msg storage.Message, lineCount int) { +func (ses *Session) sendMessageTop(msg storage.StoreMessage, lineCount int) { reader, err := msg.RawReader() if err != nil { ses.logError("Failed to read message for RETR command") @@ -522,7 +522,7 @@ func (ses *Session) processDeletes() { for i, msg := range ses.messages { if !ses.retain[i] { ses.logTrace("Deleting %v", msg) - if err := msg.Delete(); err != nil { + if err := ses.server.dataStore.RemoveMessage(ses.user, msg.ID()); err != nil { ses.logWarn("Error deleting %v: %v", msg, err) } } diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index 29098ac..d516bf3 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -16,6 +16,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/test" ) type scriptStep struct { @@ -25,10 +26,8 @@ type scriptStep struct { // Test commands in GREET state func TestGreetState(t *testing.T) { - // Setup mock objects - mds := &storage.MockDataStore{} - - server, logbuf, teardown := setupSMTPServer(mds) + ds := test.NewStore() + server, logbuf, teardown := setupSMTPServer(ds) defer teardown() // Test out some mangled HELOs @@ -82,10 +81,8 @@ func TestGreetState(t *testing.T) { // Test commands in READY state func TestReadyState(t *testing.T) { - // Setup mock objects - mds := &storage.MockDataStore{} - - server, logbuf, teardown := setupSMTPServer(mds) + ds := test.NewStore() + server, logbuf, teardown := setupSMTPServer(ds) defer teardown() // Test out some mangled READY commands diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index b325678..26c1612 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -34,7 +34,7 @@ type Message struct { // newMessage creates a new FileMessage object and sets the Date and ID fields. // It will also delete messages over messageCap if configured. -func (mb *mbox) newMessage() (storage.Message, error) { +func (mb *mbox) newMessage() (storage.StoreMessage, error) { // Load index if !mb.indexLoaded { if err := mb.readIndex(); err != nil { @@ -45,7 +45,7 @@ func (mb *mbox) newMessage() (storage.Message, error) { if mb.store.messageCap > 0 { for len(mb.messages) >= mb.store.messageCap { log.Infof("Mailbox %q over configured message cap", mb.name) - if err := mb.messages[0].Delete(); err != nil { + if err := mb.removeMessage(mb.messages[0].ID()); err != nil { log.Errorf("Error deleting message: %s", err) } } @@ -55,6 +55,11 @@ func (mb *mbox) newMessage() (storage.Message, error) { return &Message{mailbox: mb, Fid: id, Fdate: date, writable: true}, nil } +// Mailbox returns the name of the mailbox this message resides in. +func (m *Message) Mailbox() string { + return m.mailbox.name +} + // ID gets the ID of the Message func (m *Message) ID() string { return m.Fid @@ -240,29 +245,3 @@ func (m *Message) Close() error { m.mailbox.messages = append(m.mailbox.messages, m) return m.mailbox.writeIndex() } - -// Delete this Message from disk by removing it from the index and deleting the -// raw files. -func (m *Message) Delete() error { - messages := m.mailbox.messages - for i, mm := range messages { - if m == mm { - // Slice around message we are deleting - m.mailbox.messages = append(messages[:i], messages[i+1:]...) - break - } - } - if err := m.mailbox.writeIndex(); err != nil { - return err - } - - if len(m.mailbox.messages) == 0 { - // This was the last message, thus writeIndex() has removed the entire - // directory; we don't need to delete the raw file. - return nil - } - - // There are still messages in the index - log.Tracef("Deleting %v", m.rawPath()) - return os.Remove(m.rawPath()) -} diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index fb09ad0..3e5f9bb 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -75,7 +75,7 @@ func New(cfg config.DataStoreConfig) storage.Store { } // GetMessage returns the messages in the named mailbox, or an error. -func (fs *Store) GetMessage(mailbox, id string) (storage.Message, error) { +func (fs *Store) GetMessage(mailbox, id string) (storage.StoreMessage, error) { mb, err := fs.mbox(mailbox) if err != nil { return nil, err @@ -84,7 +84,7 @@ func (fs *Store) GetMessage(mailbox, id string) (storage.Message, error) { } // GetMessages returns the messages in the named mailbox, or an error. -func (fs *Store) GetMessages(mailbox string) ([]storage.Message, error) { +func (fs *Store) GetMessages(mailbox string) ([]storage.StoreMessage, error) { mb, err := fs.mbox(mailbox) if err != nil { return nil, err @@ -92,6 +92,15 @@ func (fs *Store) GetMessages(mailbox string) ([]storage.Message, error) { return mb.getMessages() } +// RemoveMessage deletes a message by ID from the specified mailbox. +func (fs *Store) RemoveMessage(mailbox, id string) error { + mb, err := fs.mbox(mailbox) + if err != nil { + return err + } + return mb.removeMessage(id) +} + // PurgeMessages deletes all messages in the named mailbox, or returns an error. func (fs *Store) PurgeMessages(mailbox string) error { mb, err := fs.mbox(mailbox) @@ -103,7 +112,7 @@ func (fs *Store) PurgeMessages(mailbox string) error { // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it // continues to return true. -func (fs *Store) VisitMailboxes(f func([]storage.Message) (cont bool)) error { +func (fs *Store) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) error { infos1, err := ioutil.ReadDir(fs.mailPath) if err != nil { return err @@ -159,7 +168,7 @@ func (fs *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { } // NewMessage is temproary until #69 MessageData refactor -func (fs *Store) NewMessage(mailbox string) (storage.Message, error) { +func (fs *Store) NewMessage(mailbox string) (storage.StoreMessage, error) { mb, err := fs.mbox(mailbox) if err != nil { return nil, err @@ -196,13 +205,13 @@ type mbox struct { // getMessages scans the mailbox directory for .gob files and decodes them into // a slice of Message objects. -func (mb *mbox) getMessages() ([]storage.Message, error) { +func (mb *mbox) getMessages() ([]storage.StoreMessage, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err } } - messages := make([]storage.Message, len(mb.messages)) + messages := make([]storage.StoreMessage, len(mb.messages)) for i, m := range mb.messages { messages[i] = m } @@ -210,7 +219,7 @@ func (mb *mbox) getMessages() ([]storage.Message, error) { } // getMessage decodes a single message by ID and returns a Message object. -func (mb *mbox) getMessage(id string) (storage.Message, error) { +func (mb *mbox) getMessage(id string) (storage.StoreMessage, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err @@ -227,6 +236,38 @@ func (mb *mbox) getMessage(id string) (storage.Message, error) { return nil, storage.ErrNotExist } +// removeMessage deletes the message off disk and removes it from the index. +func (mb *mbox) removeMessage(id string) error { + if !mb.indexLoaded { + if err := mb.readIndex(); err != nil { + return err + } + } + var msg *Message + for i, m := range mb.messages { + if id == m.ID() { + msg = m + // Slice around message we are deleting + mb.messages = append(mb.messages[:i], mb.messages[i+1:]...) + break + } + } + if msg == nil { + return storage.ErrNotExist + } + if err := mb.writeIndex(); err != nil { + return err + } + if len(mb.messages) == 0 { + // This was the last message, thus writeIndex() has removed the entire + // directory; we don't need to delete the raw file. + return nil + } + // There are still messages in the index + log.Tracef("Deleting %v", msg.rawPath()) + return os.Remove(msg.rawPath()) +} + // purge deletes all messages in this mailbox. func (mb *mbox) purge() error { mb.messages = mb.messages[:0] @@ -256,14 +297,18 @@ func (mb *mbox) readIndex() error { log.Errorf("Failed to close %q: %v", mb.indexPath, err) } }() - // Decode gob data dec := gob.NewDecoder(bufio.NewReader(file)) + name := "" + if err = dec.Decode(&name); err != nil { + return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) + } + mb.name = name for { - msg := new(Message) + // Load messages until EOF + msg := &Message{} if err = dec.Decode(msg); err != nil { if err == io.EOF { - // It's OK to get an EOF here break } return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) @@ -271,7 +316,6 @@ func (mb *mbox) readIndex() error { msg.mailbox = mb mb.messages = append(mb.messages, msg) } - mb.indexLoaded = true return nil } @@ -294,9 +338,12 @@ func (mb *mbox) writeIndex() error { writer := bufio.NewWriter(file) // Write each message and then flush enc := gob.NewEncoder(writer) + if err = enc.Encode(mb.name); err != nil { + _ = file.Close() + return err + } for _, m := range mb.messages { - err = enc.Encode(m) - if err != nil { + if err = enc.Encode(m); err != nil { _ = file.Close() return err } @@ -314,7 +361,6 @@ func (mb *mbox) writeIndex() error { log.Tracef("Removing mailbox %v", mb.path) return mb.removeDir() } - return nil } diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index a7bb039..8db7253 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -63,9 +63,7 @@ func TestFSDirStructure(t *testing.T) { assert.True(t, isFile(expect), "Expected %q to be a file", expect) // Delete message - msg, err := ds.GetMessage(mbName, id1) - assert.Nil(t, err) - err = msg.Delete() + err := ds.RemoveMessage(mbName, id1) assert.Nil(t, err) // Message should be removed @@ -75,9 +73,7 @@ func TestFSDirStructure(t *testing.T) { assert.True(t, isFile(expect), "Expected %q to be a file", expect) // Delete message - msg, err = ds.GetMessage(mbName, id2) - assert.Nil(t, err) - err = msg.Delete() + err = ds.RemoveMessage(mbName, id2) assert.Nil(t, err) // Message should be removed @@ -114,7 +110,7 @@ func TestFSVisitMailboxes(t *testing.T) { } seen := 0 - err := ds.VisitMailboxes(func(messages []storage.Message) bool { + err := ds.VisitMailboxes(func(messages []storage.StoreMessage) bool { seen++ count := len(messages) if count != 2 { @@ -196,8 +192,14 @@ func TestFSDelete(t *testing.T) { len(subjects), len(msgs)) // Delete a couple messages - _ = msgs[1].Delete() - _ = msgs[3].Delete() + err = ds.RemoveMessage(mbName, msgs[1].ID()) + if err != nil { + t.Fatal(err) + } + err = ds.RemoveMessage(mbName, msgs[3].ID()) + if err != nil { + t.Fatal(err) + } // Confirm deletion msgs, err = ds.GetMessages(mbName) diff --git a/pkg/storage/retention.go b/pkg/storage/retention.go index 6269443..2da706e 100644 --- a/pkg/storage/retention.go +++ b/pkg/storage/retention.go @@ -119,11 +119,11 @@ func (rs *RetentionScanner) DoScan() error { cutoff := time.Now().Add(-1 * rs.retentionPeriod) retained := 0 // Loop over all mailboxes. - err := rs.ds.VisitMailboxes(func(messages []Message) bool { + err := rs.ds.VisitMailboxes(func(messages []StoreMessage) bool { for _, msg := range messages { if msg.Date().Before(cutoff) { - log.Tracef("Purging expired message %v", msg.ID()) - if err := msg.Delete(); err != nil { + log.Tracef("Purging expired message %v/%v", msg.Mailbox(), msg.ID()) + if err := rs.ds.RemoveMessage(msg.Mailbox(), msg.ID()); err != nil { log.Errorf("Failed to purge message %v: %v", msg.ID(), err) } else { expRetentionDeletesTotal.Add(1) diff --git a/pkg/storage/retention_test.go b/pkg/storage/retention_test.go index bd862cf..f6ed828 100644 --- a/pkg/storage/retention_test.go +++ b/pkg/storage/retention_test.go @@ -20,11 +20,17 @@ func TestDoRetentionScan(t *testing.T) { old2 := mockMessage(12) old3 := mockMessage(24) ds.AddMessage("mb1", new1) + new1.On("Mailbox").Return("mb1") ds.AddMessage("mb1", old1) + old1.On("Mailbox").Return("mb1") ds.AddMessage("mb1", old2) + old2.On("Mailbox").Return("mb1") ds.AddMessage("mb2", old3) + old3.On("Mailbox").Return("mb2") ds.AddMessage("mb2", new2) + new2.On("Mailbox").Return("mb2") ds.AddMessage("mb3", new3) + new3.On("Mailbox").Return("mb3") // Test 4 hour retention cfg := config.DataStoreConfig{ RetentionMinutes: 239, @@ -36,13 +42,17 @@ func TestDoRetentionScan(t *testing.T) { t.Error(err) } // Delete should not have been called on new messages - new1.AssertNotCalled(t, "Delete") - new2.AssertNotCalled(t, "Delete") - new3.AssertNotCalled(t, "Delete") + for _, m := range []storage.StoreMessage{new1, new2, new3} { + if ds.MessageDeleted(m) { + t.Errorf("Expected %v to be present, was deleted", m.ID()) + } + } // Delete should have been called once on old messages - old1.AssertNumberOfCalls(t, "Delete", 1) - old2.AssertNumberOfCalls(t, "Delete", 1) - old3.AssertNumberOfCalls(t, "Delete", 1) + for _, m := range []storage.StoreMessage{old1, old2, old3} { + if !ds.MessageDeleted(m) { + t.Errorf("Expected %v to be deleted, was present", m.ID()) + } + } } // Make a MockMessage of a specific age diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 83cf635..425a792 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -12,27 +12,29 @@ import ( ) var ( - // ErrNotExist indicates the requested message does not exist - ErrNotExist = errors.New("Message does not exist") + // ErrNotExist indicates the requested message does not exist. + ErrNotExist = errors.New("message does not exist") // ErrNotWritable indicates the message is closed; no longer writable ErrNotWritable = errors.New("Message not writable") ) -// Store is an interface to get Mailboxes stored in Inbucket +// Store is the interface Inbucket uses to interact with storage implementations. type Store interface { - GetMessage(mailbox string, id string) (Message, error) - GetMessages(mailbox string) ([]Message, error) + GetMessage(mailbox, id string) (StoreMessage, error) + GetMessages(mailbox string) ([]StoreMessage, error) PurgeMessages(mailbox string) error - VisitMailboxes(f func([]Message) (cont bool)) error + RemoveMessage(mailbox, id string) error + VisitMailboxes(f func([]StoreMessage) (cont bool)) error // LockFor is a temporary hack to fix #77 until Datastore revamp LockFor(emailAddress string) (*sync.RWMutex, error) // NewMessage is temproary until #69 MessageData refactor - NewMessage(mailbox string) (Message, error) + NewMessage(mailbox string) (StoreMessage, error) } -// Message is an interface for a single message in a Mailbox -type Message interface { +// StoreMessage represents a message to be stored, or returned from a storage implementation. +type StoreMessage interface { + Mailbox() string ID() string From() string To() []string @@ -44,7 +46,6 @@ type Message interface { ReadRaw() (raw *string, err error) Append(data []byte) error Close() error - Delete() error String() string Size() int64 } diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index 6b2604d..16c40b3 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -16,15 +16,21 @@ type MockDataStore struct { } // GetMessage mock function -func (m *MockDataStore) GetMessage(name, id string) (Message, error) { +func (m *MockDataStore) GetMessage(name, id string) (StoreMessage, error) { args := m.Called(name, id) - return args.Get(0).(Message), args.Error(1) + return args.Get(0).(StoreMessage), args.Error(1) } // GetMessages mock function -func (m *MockDataStore) GetMessages(name string) ([]Message, error) { +func (m *MockDataStore) GetMessages(name string) ([]StoreMessage, error) { args := m.Called(name) - return args.Get(0).([]Message), args.Error(1) + return args.Get(0).([]StoreMessage), args.Error(1) +} + +// RemoveMessage mock function +func (m *MockDataStore) RemoveMessage(name, id string) error { + args := m.Called(name, id) + return args.Error(0) } // PurgeMessages mock function @@ -39,14 +45,14 @@ func (m *MockDataStore) LockFor(name string) (*sync.RWMutex, error) { } // NewMessage temporary for #69 -func (m *MockDataStore) NewMessage(mailbox string) (Message, error) { +func (m *MockDataStore) NewMessage(mailbox string) (StoreMessage, error) { args := m.Called(mailbox) - return args.Get(0).(Message), args.Error(1) + return args.Get(0).(StoreMessage), args.Error(1) } // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it // continues to return true. -func (m *MockDataStore) VisitMailboxes(f func([]Message) (cont bool)) error { +func (m *MockDataStore) VisitMailboxes(f func([]StoreMessage) (cont bool)) error { return nil } @@ -55,6 +61,12 @@ type MockMessage struct { mock.Mock } +// Mailbox mock function +func (m *MockMessage) Mailbox() string { + args := m.Called() + return args.String(0) +} + // ID mock function func (m *MockMessage) ID() string { args := m.Called() @@ -127,12 +139,6 @@ func (m *MockMessage) Close() error { return args.Error(0) } -// Delete mock function -func (m *MockMessage) Delete() error { - args := m.Called() - return args.Error(0) -} - // String mock function func (m *MockMessage) String() string { args := m.Called() diff --git a/pkg/test/storage.go b/pkg/test/storage.go index fab78ca..92195ff 100644 --- a/pkg/test/storage.go +++ b/pkg/test/storage.go @@ -2,6 +2,7 @@ package test import ( "errors" + "sync" "github.com/jhillyerd/inbucket/pkg/storage" ) @@ -9,24 +10,26 @@ import ( // StoreStub stubs storage.Store for testing. type StoreStub struct { storage.Store - mailboxes map[string][]storage.Message + mailboxes map[string][]storage.StoreMessage + deleted map[storage.StoreMessage]struct{} } // NewStore creates a new StoreStub. func NewStore() *StoreStub { return &StoreStub{ - mailboxes: make(map[string][]storage.Message), + mailboxes: make(map[string][]storage.StoreMessage), + deleted: make(map[storage.StoreMessage]struct{}), } } // AddMessage adds a message to the specified mailbox. -func (s *StoreStub) AddMessage(mailbox string, m storage.Message) { +func (s *StoreStub) AddMessage(mailbox string, m storage.StoreMessage) { msgs := s.mailboxes[mailbox] s.mailboxes[mailbox] = append(msgs, m) } // GetMessage gets a message by ID from the specified mailbox. -func (s *StoreStub) GetMessage(mailbox, id string) (storage.Message, error) { +func (s *StoreStub) GetMessage(mailbox, id string) (storage.StoreMessage, error) { if mailbox == "messageerr" { return nil, errors.New("internal error") } @@ -39,16 +42,36 @@ func (s *StoreStub) GetMessage(mailbox, id string) (storage.Message, error) { } // GetMessages gets all the messages for the specified mailbox. -func (s *StoreStub) GetMessages(mailbox string) ([]storage.Message, error) { +func (s *StoreStub) GetMessages(mailbox string) ([]storage.StoreMessage, error) { if mailbox == "messageserr" { return nil, errors.New("internal error") } return s.mailboxes[mailbox], nil } +// RemoveMessage deletes a message by ID from the specified mailbox. +func (s *StoreStub) RemoveMessage(mailbox, id string) error { + mb, ok := s.mailboxes[mailbox] + if ok { + var msg storage.StoreMessage + for i, m := range mb { + if m.ID() == id { + msg = m + s.mailboxes[mailbox] = append(mb[:i], mb[i+1:]...) + break + } + } + if msg != nil { + s.deleted[msg] = struct{}{} + return nil + } + } + return storage.ErrNotExist +} + // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it // continues to return true. -func (s *StoreStub) VisitMailboxes(f func([]storage.Message) (cont bool)) error { +func (s *StoreStub) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) error { for _, v := range s.mailboxes { if !f(v) { return nil @@ -58,6 +81,18 @@ func (s *StoreStub) VisitMailboxes(f func([]storage.Message) (cont bool)) error } // NewMessage is temproary until #69 MessageData refactor -func (s *StoreStub) NewMessage(mailbox string) (storage.Message, error) { +func (s *StoreStub) NewMessage(mailbox string) (storage.StoreMessage, error) { return nil, nil } + +// LockFor mock function returns a new RWMutex, never errors. +// TODO(#69) remove +func (s *StoreStub) LockFor(name string) (*sync.RWMutex, error) { + return &sync.RWMutex{}, nil +} + +// MessageDeleted returns true if the specified message was deleted +func (s *StoreStub) MessageDeleted(m storage.StoreMessage) bool { + _, ok := s.deleted[m] + return ok +} From 3bc66d278893447f1b941ca69517670c94088d80 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 11 Mar 2018 16:56:09 -0700 Subject: [PATCH 07/26] storage: Store addresses as mail.Address for #69 --- pkg/rest/apiv1_controller.go | 8 ++++---- pkg/rest/apiv1_controller_test.go | 10 +++++----- pkg/rest/testutils_test.go | 9 +++++++-- pkg/server/smtp/handler.go | 4 ++-- pkg/server/smtp/handler_test.go | 9 +++++---- pkg/storage/file/fmessage.go | 18 ++++++++---------- pkg/storage/storage.go | 4 ++-- pkg/storage/testing.go | 8 ++++---- pkg/stringutil/utils.go | 12 ++++++++++++ 9 files changed, 49 insertions(+), 33 deletions(-) diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index 6424648..bc03bfd 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -36,8 +36,8 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( jmessages[i] = &model.JSONMessageHeaderV1{ Mailbox: name, ID: msg.ID(), - From: msg.From(), - To: msg.To(), + From: msg.From().String(), + To: stringutil.StringAddressList(msg.To()), Subject: msg.Subject(), Date: msg.Date(), Size: msg.Size(), @@ -90,8 +90,8 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( &model.JSONMessageV1{ Mailbox: name, ID: msg.ID(), - From: msg.From(), - To: msg.To(), + From: msg.From().String(), + To: stringutil.StringAddressList(msg.To()), Subject: msg.Subject(), Date: msg.Date(), Size: msg.Size(), diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index a48fd38..8053a48 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -67,16 +67,16 @@ func TestRestMailboxList(t *testing.T) { data1 := &InputMessageData{ Mailbox: "good", ID: "0001", - From: "from1", - To: []string{"to1"}, + From: "", + To: []string{""}, Subject: "subject 1", Date: time.Date(2012, 2, 1, 10, 11, 12, 253, time.FixedZone("PST", -800)), } data2 := &InputMessageData{ Mailbox: "good", ID: "0002", - From: "from2", - To: []string{"to1"}, + From: "", + To: []string{""}, Subject: "subject 2", Date: time.Date(2012, 7, 1, 10, 11, 12, 253, time.FixedZone("PDT", -700)), } @@ -171,7 +171,7 @@ func TestRestMessage(t *testing.T) { data1 := &InputMessageData{ Mailbox: "good", ID: "0001", - From: "from1", + From: "", Subject: "subject 1", Date: time.Date(2012, 2, 1, 10, 11, 12, 253, time.FixedZone("PST", -800)), Header: mail.Header{ diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index ef294df..4b7d669 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -26,10 +26,15 @@ type InputMessageData struct { } func (d *InputMessageData) MockMessage() *storage.MockMessage { + from, _ := mail.ParseAddress(d.From) + to := make([]*mail.Address, len(d.To)) + for i, a := range d.To { + to[i], _ = mail.ParseAddress(a) + } msg := &storage.MockMessage{} msg.On("ID").Return(d.ID) - msg.On("From").Return(d.From) - msg.On("To").Return(d.To) + msg.On("From").Return(from) + msg.On("To").Return(to) msg.On("Subject").Return(d.Subject) msg.On("Date").Return(d.Date) msg.On("Size").Return(d.Size) diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 50bf9ef..b1a6eb3 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -479,8 +479,8 @@ func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) broadcast := msghub.Message{ Mailbox: name, ID: msg.ID(), - From: msg.From(), - To: msg.To(), + From: msg.From().String(), + To: stringutil.StringAddressList(msg.To()), Subject: msg.Subject(), Date: msg.Date(), Size: msg.Size(), diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index d516bf3..ca9ff2e 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -8,6 +8,7 @@ import ( "log" "net" + "net/mail" "net/textproto" "os" "testing" @@ -145,8 +146,8 @@ func TestMailState(t *testing.T) { msg1 := &storage.MockMessage{} mds.On("NewMessage", "u1").Return(msg1, nil) msg1.On("ID").Return("") - msg1.On("From").Return("") - msg1.On("To").Return(make([]string, 0)) + msg1.On("From").Return(&mail.Address{}) + msg1.On("To").Return(make([]*mail.Address, 0)) msg1.On("Date").Return(time.Time{}) msg1.On("Subject").Return("") msg1.On("Size").Return(0) @@ -257,8 +258,8 @@ func TestDataState(t *testing.T) { msg1 := &storage.MockMessage{} mds.On("NewMessage", "u1").Return(msg1, nil) msg1.On("ID").Return("") - msg1.On("From").Return("") - msg1.On("To").Return(make([]string, 0)) + msg1.On("From").Return(&mail.Address{}) + msg1.On("To").Return(make([]*mail.Address, 0)) msg1.On("Date").Return(time.Time{}) msg1.On("Subject").Return("") msg1.On("Size").Return(0) diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index 26c1612..3f8b0d7 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -22,8 +22,8 @@ type Message struct { // Stored in GOB Fid string Fdate time.Time - Ffrom string - Fto []string + Ffrom *mail.Address + Fto []*mail.Address Fsubject string Fsize int64 // These are for creating new messages only @@ -71,12 +71,12 @@ func (m *Message) Date() time.Time { } // From returns the value of the Message From header -func (m *Message) From() string { +func (m *Message) From() *mail.Address { return m.Ffrom } // To returns the value of the Message To header -func (m *Message) To() []string { +func (m *Message) To() []*mail.Address { return m.Fto } @@ -220,19 +220,17 @@ func (m *Message) Close() error { // Only public fields are stored in gob, hence starting with capital F // Parse From address if address, err := mail.ParseAddress(body.GetHeader("From")); err == nil { - m.Ffrom = address.String() + m.Ffrom = address } else { - m.Ffrom = body.GetHeader("From") + m.Ffrom = &mail.Address{Address: body.GetHeader("From")} } m.Fsubject = body.GetHeader("Subject") // Turn the To header into a slice if addresses, err := body.AddressList("To"); err == nil { - for _, a := range addresses { - m.Fto = append(m.Fto, a.String()) - } + m.Fto = addresses } else { - m.Fto = []string{body.GetHeader("To")} + m.Fto = []*mail.Address{{Address: body.GetHeader("To")}} } // Refresh the index before adding our message diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 425a792..c3d2019 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -36,8 +36,8 @@ type Store interface { type StoreMessage interface { Mailbox() string ID() string - From() string - To() []string + From() *mail.Address + To() []*mail.Address Date() time.Time Subject() string RawReader() (reader io.ReadCloser, err error) diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index 16c40b3..9defa55 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -74,15 +74,15 @@ func (m *MockMessage) ID() string { } // From mock function -func (m *MockMessage) From() string { +func (m *MockMessage) From() *mail.Address { args := m.Called() - return args.String(0) + return args.Get(0).(*mail.Address) } // To mock function -func (m *MockMessage) To() []string { +func (m *MockMessage) To() []*mail.Address { args := m.Called() - return args.Get(0).([]string) + return args.Get(0).([]*mail.Address) } // Date mock function diff --git a/pkg/stringutil/utils.go b/pkg/stringutil/utils.go index 450ae1f..193e552 100644 --- a/pkg/stringutil/utils.go +++ b/pkg/stringutil/utils.go @@ -5,6 +5,7 @@ import ( "crypto/sha1" "fmt" "io" + "net/mail" "strings" ) @@ -224,3 +225,14 @@ LOOP: return buf.String(), domain, nil } + +// StringAddressList converts a list of addresses to a list of strings +func StringAddressList(addrs []*mail.Address) []string { + s := make([]string, len(addrs)) + for i, a := range addrs { + if a != nil { + s[i] = a.String() + } + } + return s +} From 10bc07a18e5cba41c4bc3523c1bd5fda467f6fd7 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 11 Mar 2018 22:25:21 -0700 Subject: [PATCH 08/26] message: Implement service layer, stubs for #81 I've made some effort to wire the manager into the controllers, but tests are currently failing. --- cmd/inbucket/main.go | 4 +- pkg/message/manager.go | 86 +++++++++++++++++++++++++++++++ pkg/message/message.go | 26 ++++++++++ pkg/rest/apiv1_controller.go | 60 +++++++++------------ pkg/rest/apiv1_controller_test.go | 75 ++++++++++++++++++++++++--- pkg/rest/model/apiv1_model.go | 3 +- pkg/rest/testutils_test.go | 5 +- pkg/server/web/context.go | 3 ++ pkg/server/web/server.go | 4 ++ pkg/storage/file/fmessage.go | 16 ------ pkg/storage/file/fstore_test.go | 2 - pkg/storage/storage.go | 1 - pkg/test/manager.go | 53 +++++++++++++++++++ pkg/webui/mailbox_controller.go | 64 ++++++++--------------- 14 files changed, 291 insertions(+), 111 deletions(-) create mode 100644 pkg/message/manager.go create mode 100644 pkg/message/message.go create mode 100644 pkg/test/manager.go diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index 9f1d116..ff01d3f 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -14,6 +14,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/rest" "github.com/jhillyerd/inbucket/pkg/server/pop3" @@ -123,7 +124,8 @@ func main() { retentionScanner.Start() // Start HTTP server - web.Initialize(config.GetWebConfig(), shutdownChan, ds, msgHub) + mm := &message.StoreManager{Store: ds} + web.Initialize(config.GetWebConfig(), shutdownChan, mm, ds, msgHub) webui.SetupRoutes(web.Router) rest.SetupRoutes(web.Router) go web.Start(rootCtx) diff --git a/pkg/message/manager.go b/pkg/message/manager.go new file mode 100644 index 0000000..5782273 --- /dev/null +++ b/pkg/message/manager.go @@ -0,0 +1,86 @@ +package message + +import ( + "io" + + "github.com/jhillyerd/enmime" + "github.com/jhillyerd/inbucket/pkg/storage" +) + +// Manager is the interface controllers use to interact with messages. +type Manager interface { + GetMetadata(mailbox string) ([]*Metadata, error) + GetMessage(mailbox, id string) (*Message, error) + PurgeMessages(mailbox string) error + RemoveMessage(mailbox, id string) error + SourceReader(mailbox, id string) (io.ReadCloser, error) +} + +// StoreManager is a message Manager backed by the storage.Store. +type StoreManager struct { + Store storage.Store +} + +// GetMetadata returns a slice of metadata for the specified mailbox. +func (s *StoreManager) GetMetadata(mailbox string) ([]*Metadata, error) { + messages, err := s.Store.GetMessages(mailbox) + if err != nil { + return nil, err + } + metas := make([]*Metadata, len(messages)) + for i, sm := range messages { + metas[i] = makeMetadata(sm) + } + return metas, nil +} + +// GetMessage returns the specified message. +func (s *StoreManager) GetMessage(mailbox, id string) (*Message, error) { + sm, err := s.Store.GetMessage(mailbox, id) + if err != nil { + return nil, err + } + r, err := sm.RawReader() + if err != nil { + return nil, err + } + env, err := enmime.ReadEnvelope(r) + if err != nil { + return nil, err + } + _ = r.Close() + header := makeMetadata(sm) + return &Message{Metadata: *header, Envelope: env}, nil +} + +// PurgeMessages removes all messages from the specified mailbox. +func (s *StoreManager) PurgeMessages(mailbox string) error { + return s.Store.PurgeMessages(mailbox) +} + +// RemoveMessage deletes the specified message. +func (s *StoreManager) RemoveMessage(mailbox, id string) error { + return s.Store.RemoveMessage(mailbox, id) +} + +// SourceReader allows the stored message source to be read. +func (s *StoreManager) SourceReader(mailbox, id string) (io.ReadCloser, error) { + sm, err := s.Store.GetMessage(mailbox, id) + if err != nil { + return nil, err + } + return sm.RawReader() +} + +// makeMetadata populates Metadata from a StoreMessage. +func makeMetadata(m storage.StoreMessage) *Metadata { + return &Metadata{ + Mailbox: m.Mailbox(), + ID: m.ID(), + From: m.From(), + To: m.To(), + Date: m.Date(), + Subject: m.Subject(), + Size: m.Size(), + } +} diff --git a/pkg/message/message.go b/pkg/message/message.go new file mode 100644 index 0000000..fd5f25b --- /dev/null +++ b/pkg/message/message.go @@ -0,0 +1,26 @@ +// Package message contains message handling logic. +package message + +import ( + "net/mail" + "time" + + "github.com/jhillyerd/enmime" +) + +// Metadata holds information about a message, but not the content. +type Metadata struct { + Mailbox string + ID string + From *mail.Address + To []*mail.Address + Date time.Time + Subject string + Size int64 +} + +// Message holds both the metadata and content of a message. +type Message struct { + Metadata + Envelope *enmime.Envelope +} diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index bc03bfd..135669e 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -24,7 +24,7 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - messages, err := ctx.DataStore.GetMessages(name) + messages, err := ctx.MsgSvc.GetMetadata(name) if err != nil { // This doesn't indicate empty, likely an IO error return fmt.Errorf("Failed to get messages for %v: %v", name, err) @@ -35,12 +35,12 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( for i, msg := range messages { jmessages[i] = &model.JSONMessageHeaderV1{ Mailbox: name, - ID: msg.ID(), - From: msg.From().String(), - To: stringutil.StringAddressList(msg.To()), - Subject: msg.Subject(), - Date: msg.Date(), - Size: msg.Size(), + ID: msg.ID, + From: msg.From.String(), + To: stringutil.StringAddressList(msg.To), + Subject: msg.Subject, + Date: msg.Date, + Size: msg.Size, } } return web.RenderJSON(w, jmessages) @@ -54,7 +54,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - msg, err := ctx.DataStore.GetMessage(name, id) + msg, err := ctx.MsgSvc.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -63,14 +63,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - header, err := msg.ReadHeader() - if err != nil { - return fmt.Errorf("ReadHeader(%q) failed: %v", id, err) - } - mime, err := msg.ReadBody() - if err != nil { - return fmt.Errorf("ReadBody(%q) failed: %v", id, err) - } + mime := msg.Envelope attachments := make([]*model.JSONMessageAttachmentV1, len(mime.Attachments)) for i, att := range mime.Attachments { @@ -89,13 +82,13 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( return web.RenderJSON(w, &model.JSONMessageV1{ Mailbox: name, - ID: msg.ID(), - From: msg.From().String(), - To: stringutil.StringAddressList(msg.To()), - Subject: msg.Subject(), - Date: msg.Date(), - Size: msg.Size(), - Header: header.Header, + ID: msg.ID, + From: msg.From.String(), + To: stringutil.StringAddressList(msg.To), + Subject: msg.Subject, + Date: msg.Date, + Size: msg.Size, + Header: mime.Root.Header, Body: &model.JSONMessageBodyV1{ Text: mime.Text, HTML: mime.HTML, @@ -112,7 +105,7 @@ func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) return err } // Delete all messages - err = ctx.DataStore.PurgeMessages(name) + err = ctx.MsgSvc.PurgeMessages(name) if err != nil { return fmt.Errorf("Mailbox(%q) purge failed: %v", name, err) } @@ -129,25 +122,20 @@ func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return err } - message, err := ctx.DataStore.GetMessage(name, id) + + r, err := ctx.MsgSvc.SourceReader(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil } if err != nil { // This doesn't indicate missing, likely an IO error - return fmt.Errorf("GetMessage(%q) failed: %v", id, err) + return fmt.Errorf("SourceReader(%q) failed: %v", id, err) } - raw, err := message.ReadRaw() - if err != nil { - return fmt.Errorf("ReadRaw(%q) failed: %v", id, err) - } - + // Output message source w.Header().Set("Content-Type", "text/plain") - if _, err := io.WriteString(w, *raw); err != nil { - return err - } - return nil + _, err = io.Copy(w, r) + return err } // MailboxDeleteV1 removes a particular message from a mailbox @@ -158,7 +146,7 @@ func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return err } - err = ctx.DataStore.RemoveMessage(name, id) + err = ctx.MsgSvc.RemoveMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index 8053a48..48d2841 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -4,10 +4,14 @@ import ( "encoding/json" "io" "net/mail" + "net/textproto" "os" + "strings" "testing" "time" + "github.com/jhillyerd/enmime" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/test" ) @@ -31,7 +35,8 @@ const ( func TestRestMailboxList(t *testing.T) { // Setup ds := test.NewStore() - logbuf := setupWebServer(ds) + mm := test.NewManager() + logbuf := setupWebServer(mm, ds) // Test invalid mailbox name w, err := testRestGet(baseURL + "/mailbox/foo@bar") @@ -80,10 +85,24 @@ func TestRestMailboxList(t *testing.T) { Subject: "subject 2", Date: time.Date(2012, 7, 1, 10, 11, 12, 253, time.FixedZone("PDT", -700)), } - msg1 := data1.MockMessage() - msg2 := data2.MockMessage() - ds.AddMessage("good", msg1) - ds.AddMessage("good", msg2) + meta1 := message.Metadata{ + Mailbox: "good", + ID: "0001", + From: &mail.Address{Name: "", Address: "from1@host"}, + To: []*mail.Address{{Name: "", Address: "to1@host"}}, + Subject: "subject 1", + Date: time.Date(2012, 2, 1, 10, 11, 12, 253, time.FixedZone("PST", -800)), + } + meta2 := message.Metadata{ + Mailbox: "good", + ID: "0002", + From: &mail.Address{Name: "", Address: "from2@host"}, + To: []*mail.Address{{Name: "", Address: "to1@host"}}, + Subject: "subject 2", + Date: time.Date(2012, 7, 1, 10, 11, 12, 253, time.FixedZone("PDT", -700)), + } + mm.AddMessage("good", &message.Message{Metadata: meta1}) + mm.AddMessage("good", &message.Message{Metadata: meta2}) // Check return code w, err = testRestGet(baseURL + "/mailbox/good") @@ -96,6 +115,25 @@ func TestRestMailboxList(t *testing.T) { } // Check JSON + got := w.Body.String() + testStrings := []string{ + `{"mailbox":"good","id":"0001","from":"\u003cfrom1@host\u003e",` + + `"to":["\u003cto1@host\u003e"],"subject":"subject 1",` + + `"date":"2012-02-01T10:11:12.000000253-00:13","size":0}`, + `{"mailbox":"good","id":"0002","from":"\u003cfrom2@host\u003e",` + + `"to":["\u003cto1@host\u003e"],"subject":"subject 2",` + + `"date":"2012-07-01T10:11:12.000000253-00:11","size":0}`, + } + for _, ts := range testStrings { + t.Run(ts, func(t *testing.T) { + if !strings.Contains(got, ts) { + t.Errorf("got:\n%s\nwant to contain:\n%s", got, ts) + } + }) + } + + // Check JSON + // TODO transitional while refactoring dec := json.NewDecoder(w.Body) var result []interface{} if err := dec.Decode(&result); err != nil { @@ -128,7 +166,8 @@ func TestRestMailboxList(t *testing.T) { func TestRestMessage(t *testing.T) { // Setup ds := test.NewStore() - logbuf := setupWebServer(ds) + mm := test.NewManager() + logbuf := setupWebServer(mm, ds) // Test invalid mailbox name w, err := testRestGet(baseURL + "/mailbox/foo@bar/0001") @@ -168,6 +207,26 @@ func TestRestMessage(t *testing.T) { } // Test JSON message headers + msg1 := &message.Message{ + Metadata: message.Metadata{ + Mailbox: "good", + ID: "0001", + From: &mail.Address{Name: "", Address: "from1@host"}, + To: []*mail.Address{{Name: "", Address: "to1@host"}}, + Subject: "subject 1", + Date: time.Date(2012, 2, 1, 10, 11, 12, 253, time.FixedZone("PST", -800)), + }, + Envelope: &enmime.Envelope{ + Text: "This is some text", + HTML: "This is some HTML", + Root: &enmime.Part{ + Header: textproto.MIMEHeader{ + "To": []string{"fred@fish.com", "keyword@nsa.gov"}, + "From": []string{"noreply@inbucket.org"}, + }, + }, + }, + } data1 := &InputMessageData{ Mailbox: "good", ID: "0001", @@ -181,8 +240,7 @@ func TestRestMessage(t *testing.T) { Text: "This is some text", HTML: "This is some HTML", } - msg1 := data1.MockMessage() - ds.AddMessage("good", msg1) + mm.AddMessage("good", msg1) // Check return code w, err = testRestGet(baseURL + "/mailbox/good/0001") @@ -195,6 +253,7 @@ func TestRestMessage(t *testing.T) { } // Check JSON + // TODO transitional while refactoring dec := json.NewDecoder(w.Body) var result map[string]interface{} if err := dec.Decode(&result); err != nil { diff --git a/pkg/rest/model/apiv1_model.go b/pkg/rest/model/apiv1_model.go index 1e4f7f0..7e1e083 100644 --- a/pkg/rest/model/apiv1_model.go +++ b/pkg/rest/model/apiv1_model.go @@ -1,7 +1,6 @@ package model import ( - "net/mail" "time" ) @@ -26,7 +25,7 @@ type JSONMessageV1 struct { Date time.Time `json:"date"` Size int64 `json:"size"` Body *JSONMessageBodyV1 `json:"body"` - Header mail.Header `json:"header"` + Header map[string][]string `json:"header"` Attachments []*JSONMessageAttachmentV1 `json:"attachments"` } diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index 4b7d669..4bdcae5 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -11,6 +11,7 @@ import ( "github.com/jhillyerd/enmime" "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/server/web" "github.com/jhillyerd/inbucket/pkg/storage" @@ -193,7 +194,7 @@ func testRestGet(url string) (*httptest.ResponseRecorder, error) { return w, nil } -func setupWebServer(ds storage.Store) *bytes.Buffer { +func setupWebServer(mm message.Manager, ds storage.Store) *bytes.Buffer { // Capture log output buf := new(bytes.Buffer) log.SetOutput(buf) @@ -205,7 +206,7 @@ func setupWebServer(ds storage.Store) *bytes.Buffer { PublicDir: "../themes/bootstrap/public", } shutdownChan := make(chan bool) - web.Initialize(cfg, shutdownChan, ds, &msghub.Hub{}) + web.Initialize(cfg, shutdownChan, mm, ds, &msghub.Hub{}) SetupRoutes(web.Router) return buf diff --git a/pkg/server/web/context.go b/pkg/server/web/context.go index 422fdb5..af5f6b2 100644 --- a/pkg/server/web/context.go +++ b/pkg/server/web/context.go @@ -7,6 +7,7 @@ import ( "github.com/gorilla/mux" "github.com/gorilla/sessions" "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/storage" ) @@ -17,6 +18,7 @@ type Context struct { Session *sessions.Session DataStore storage.Store MsgHub *msghub.Hub + MsgSvc message.Manager WebConfig config.WebConfig IsJSON bool } @@ -61,6 +63,7 @@ func NewContext(req *http.Request) (*Context, error) { Session: sess, DataStore: DataStore, MsgHub: msgHub, + MsgSvc: msgSvc, WebConfig: webConfig, IsJSON: headerMatch(req, "Accept", "application/json"), } diff --git a/pkg/server/web/server.go b/pkg/server/web/server.go index 5c82430..c76a2f7 100644 --- a/pkg/server/web/server.go +++ b/pkg/server/web/server.go @@ -14,6 +14,7 @@ import ( "github.com/gorilla/sessions" "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/storage" ) @@ -27,6 +28,7 @@ var ( // msgHub holds a reference to the message pub/sub system msgHub *msghub.Hub + msgSvc message.Manager // Router is shared between httpd, webui and rest packages. It sends // incoming requests to the correct handler function @@ -51,6 +53,7 @@ func init() { func Initialize( cfg config.WebConfig, shutdownChan chan bool, + mm message.Manager, ds storage.Store, mh *msghub.Hub) { @@ -60,6 +63,7 @@ func Initialize( // NewContext() will use this DataStore for the web handlers DataStore = ds msgHub = mh + msgSvc = mm // Content Paths log.Infof("HTTP templates mapped to %q", cfg.TemplateDir) diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index 3f8b0d7..e9df65d 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -99,22 +99,6 @@ func (m *Message) rawPath() string { return filepath.Join(m.mailbox.path, m.Fid+".raw") } -// ReadHeader opens the .raw portion of a Message and returns a standard Go mail.Message object -func (m *Message) ReadHeader() (msg *mail.Message, err error) { - file, err := os.Open(m.rawPath()) - if err != nil { - return nil, err - } - defer func() { - if err := file.Close(); err != nil { - log.Errorf("Failed to close %q: %v", m.rawPath(), err) - } - }() - - reader := bufio.NewReader(file) - return mail.ReadMessage(reader) -} - // ReadBody opens the .raw portion of a Message and returns a MIMEBody object func (m *Message) ReadBody() (body *enmime.Envelope, err error) { file, err := os.Open(m.rawPath()) diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 8db7253..eb5add1 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -337,8 +337,6 @@ func TestFSMissing(t *testing.T) { assert.Nil(t, err) // Try to read parts of message - _, err = msg.ReadHeader() - assert.Error(t, err) _, err = msg.ReadBody() assert.Error(t, err) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index c3d2019..52e50cc 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -41,7 +41,6 @@ type StoreMessage interface { Date() time.Time Subject() string RawReader() (reader io.ReadCloser, err error) - ReadHeader() (msg *mail.Message, err error) ReadBody() (body *enmime.Envelope, err error) ReadRaw() (raw *string, err error) Append(data []byte) error diff --git a/pkg/test/manager.go b/pkg/test/manager.go new file mode 100644 index 0000000..47c87a8 --- /dev/null +++ b/pkg/test/manager.go @@ -0,0 +1,53 @@ +package test + +import ( + "errors" + + "github.com/jhillyerd/inbucket/pkg/message" + "github.com/jhillyerd/inbucket/pkg/storage" +) + +// ManagerStub is a test stub for message.Manager +type ManagerStub struct { + message.Manager + mailboxes map[string][]*message.Message +} + +// NewManager creates a new ManagerStub. +func NewManager() *ManagerStub { + return &ManagerStub{ + mailboxes: make(map[string][]*message.Message), + } +} + +// AddMessage adds a message to the specified mailbox. +func (m *ManagerStub) AddMessage(mailbox string, msg *message.Message) { + messages := m.mailboxes[mailbox] + m.mailboxes[mailbox] = append(messages, msg) +} + +// GetMessage gets a message by ID from the specified mailbox. +func (m *ManagerStub) GetMessage(mailbox, id string) (*message.Message, error) { + if mailbox == "messageerr" { + return nil, errors.New("internal error") + } + for _, msg := range m.mailboxes[mailbox] { + if msg.ID == id { + return msg, nil + } + } + return nil, storage.ErrNotExist +} + +// GetMetadata gets all the metadata for the specified mailbox. +func (m *ManagerStub) GetMetadata(mailbox string) ([]*message.Metadata, error) { + if mailbox == "messageserr" { + return nil, errors.New("internal error") + } + messages := m.mailboxes[mailbox] + metas := make([]*message.Metadata, len(messages)) + for i, msg := range messages { + metas[i] = &msg.Metadata + } + return metas, nil +} diff --git a/pkg/webui/mailbox_controller.go b/pkg/webui/mailbox_controller.go index 69b3a3f..ed092e7 100644 --- a/pkg/webui/mailbox_controller.go +++ b/pkg/webui/mailbox_controller.go @@ -72,7 +72,7 @@ func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - messages, err := ctx.DataStore.GetMessages(name) + messages, err := ctx.MsgSvc.GetMetadata(name) if err != nil { // This doesn't indicate empty, likely an IO error return fmt.Errorf("Failed to get messages for %v: %v", name, err) @@ -94,7 +94,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - msg, err := ctx.DataStore.GetMessage(name, id) + msg, err := ctx.MsgSvc.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -103,10 +103,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - mime, err := msg.ReadBody() - if err != nil { - return fmt.Errorf("ReadBody(%q) failed: %v", id, err) - } + mime := msg.Envelope body := template.HTML(web.TextToHTML(mime.Text)) htmlAvailable := mime.HTML != "" var htmlBody template.HTML @@ -138,25 +135,22 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - message, err := ctx.DataStore.GetMessage(name, id) + msg, err := ctx.MsgSvc.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil } if err != nil { - // This doesn't indicate missing, likely an IO error + // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - mime, err := message.ReadBody() - if err != nil { - return fmt.Errorf("ReadBody(%q) failed: %v", id, err) - } + mime := msg.Envelope // Render partial template w.Header().Set("Content-Type", "text/html; charset=UTF-8") return web.RenderPartial("mailbox/_html.html", w, map[string]interface{}{ "ctx": ctx, "name": name, - "message": message, + "message": msg, "body": template.HTML(mime.HTML), }) } @@ -169,25 +163,19 @@ func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - message, err := ctx.DataStore.GetMessage(name, id) + r, err := ctx.MsgSvc.SourceReader(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil } if err != nil { // This doesn't indicate missing, likely an IO error - return fmt.Errorf("GetMessage(%q) failed: %v", id, err) - } - raw, err := message.ReadRaw() - if err != nil { - return fmt.Errorf("ReadRaw(%q) failed: %v", id, err) + return fmt.Errorf("SourceReader(%q) failed: %v", id, err) } // Output message source w.Header().Set("Content-Type", "text/plain") - if _, err := io.WriteString(w, *raw); err != nil { - return err - } - return nil + _, err = io.Copy(w, r) + return err } // MailboxDownloadAttach sends the attachment to the client; disposition: @@ -210,19 +198,16 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Co http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - message, err := ctx.DataStore.GetMessage(name, id) + msg, err := ctx.MsgSvc.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil } if err != nil { - // This doesn't indicate missing, likely an IO error + // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - body, err := message.ReadBody() - if err != nil { - return err - } + body := msg.Envelope if int(num) >= len(body.Attachments) { ctx.Session.AddFlash("Attachment number too high", "errors") _ = ctx.Session.Save(req, w) @@ -233,10 +218,8 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Co // Output attachment w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Disposition", "attachment") - if _, err := io.Copy(w, part); err != nil { - return err - } - return nil + _, err = io.Copy(w, part) + return err } // MailboxViewAttach sends the attachment to the client for online viewing @@ -258,19 +241,16 @@ func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Contex http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - message, err := ctx.DataStore.GetMessage(name, id) + msg, err := ctx.MsgSvc.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil } if err != nil { - // This doesn't indicate missing, likely an IO error + // This doesn't indicate empty, likely an IO error return fmt.Errorf("GetMessage(%q) failed: %v", id, err) } - body, err := message.ReadBody() - if err != nil { - return err - } + body := msg.Envelope if int(num) >= len(body.Attachments) { ctx.Session.AddFlash("Attachment number too high", "errors") _ = ctx.Session.Save(req, w) @@ -280,8 +260,6 @@ func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Contex part := body.Attachments[num] // Output attachment w.Header().Set("Content-Type", part.ContentType) - if _, err := io.Copy(w, part); err != nil { - return err - } - return nil + _, err = io.Copy(w, part) + return err } From 219862797e0eed58b82cc88f92b4e6944fdd10ab Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Mon, 12 Mar 2018 20:49:06 -0700 Subject: [PATCH 09/26] web: remove DataStore from Context and controllers for #81 --- cmd/inbucket/main.go | 2 +- pkg/rest/apiv1_controller.go | 10 +++++----- pkg/rest/apiv1_controller_test.go | 6 ++---- pkg/rest/testutils_test.go | 4 ++-- pkg/server/web/context.go | 7 ++----- pkg/server/web/server.go | 12 +++--------- pkg/webui/mailbox_controller.go | 12 ++++++------ 7 files changed, 21 insertions(+), 32 deletions(-) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index ff01d3f..7020676 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -125,7 +125,7 @@ func main() { // Start HTTP server mm := &message.StoreManager{Store: ds} - web.Initialize(config.GetWebConfig(), shutdownChan, mm, ds, msgHub) + web.Initialize(config.GetWebConfig(), shutdownChan, mm, msgHub) webui.SetupRoutes(web.Router) rest.SetupRoutes(web.Router) go web.Start(rootCtx) diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index 135669e..7b40201 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -24,7 +24,7 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - messages, err := ctx.MsgSvc.GetMetadata(name) + messages, err := ctx.Manager.GetMetadata(name) if err != nil { // This doesn't indicate empty, likely an IO error return fmt.Errorf("Failed to get messages for %v: %v", name, err) @@ -54,7 +54,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - msg, err := ctx.MsgSvc.GetMessage(name, id) + msg, err := ctx.Manager.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -105,7 +105,7 @@ func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) return err } // Delete all messages - err = ctx.MsgSvc.PurgeMessages(name) + err = ctx.Manager.PurgeMessages(name) if err != nil { return fmt.Errorf("Mailbox(%q) purge failed: %v", name, err) } @@ -123,7 +123,7 @@ func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) return err } - r, err := ctx.MsgSvc.SourceReader(name, id) + r, err := ctx.Manager.SourceReader(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -146,7 +146,7 @@ func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) if err != nil { return err } - err = ctx.MsgSvc.RemoveMessage(name, id) + err = ctx.Manager.RemoveMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index 48d2841..b31cfcb 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -34,9 +34,8 @@ const ( func TestRestMailboxList(t *testing.T) { // Setup - ds := test.NewStore() mm := test.NewManager() - logbuf := setupWebServer(mm, ds) + logbuf := setupWebServer(mm) // Test invalid mailbox name w, err := testRestGet(baseURL + "/mailbox/foo@bar") @@ -165,9 +164,8 @@ func TestRestMailboxList(t *testing.T) { func TestRestMessage(t *testing.T) { // Setup - ds := test.NewStore() mm := test.NewManager() - logbuf := setupWebServer(mm, ds) + logbuf := setupWebServer(mm) // Test invalid mailbox name w, err := testRestGet(baseURL + "/mailbox/foo@bar/0001") diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index 4bdcae5..cfddede 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -194,7 +194,7 @@ func testRestGet(url string) (*httptest.ResponseRecorder, error) { return w, nil } -func setupWebServer(mm message.Manager, ds storage.Store) *bytes.Buffer { +func setupWebServer(mm message.Manager) *bytes.Buffer { // Capture log output buf := new(bytes.Buffer) log.SetOutput(buf) @@ -206,7 +206,7 @@ func setupWebServer(mm message.Manager, ds storage.Store) *bytes.Buffer { PublicDir: "../themes/bootstrap/public", } shutdownChan := make(chan bool) - web.Initialize(cfg, shutdownChan, mm, ds, &msghub.Hub{}) + web.Initialize(cfg, shutdownChan, mm, &msghub.Hub{}) SetupRoutes(web.Router) return buf diff --git a/pkg/server/web/context.go b/pkg/server/web/context.go index af5f6b2..9ec2ab8 100644 --- a/pkg/server/web/context.go +++ b/pkg/server/web/context.go @@ -9,16 +9,14 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" - "github.com/jhillyerd/inbucket/pkg/storage" ) // Context is passed into every request handler function type Context struct { Vars map[string]string Session *sessions.Session - DataStore storage.Store MsgHub *msghub.Hub - MsgSvc message.Manager + Manager message.Manager WebConfig config.WebConfig IsJSON bool } @@ -61,9 +59,8 @@ func NewContext(req *http.Request) (*Context, error) { ctx := &Context{ Vars: vars, Session: sess, - DataStore: DataStore, MsgHub: msgHub, - MsgSvc: msgSvc, + Manager: manager, WebConfig: webConfig, IsJSON: headerMatch(req, "Accept", "application/json"), } diff --git a/pkg/server/web/server.go b/pkg/server/web/server.go index c76a2f7..b3e8610 100644 --- a/pkg/server/web/server.go +++ b/pkg/server/web/server.go @@ -16,19 +16,15 @@ import ( "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" - "github.com/jhillyerd/inbucket/pkg/storage" ) // Handler is a function type that handles an HTTP request in Inbucket type Handler func(http.ResponseWriter, *http.Request, *Context) error var ( - // DataStore is where all the mailboxes and messages live - DataStore storage.Store - // msgHub holds a reference to the message pub/sub system - msgHub *msghub.Hub - msgSvc message.Manager + msgHub *msghub.Hub + manager message.Manager // Router is shared between httpd, webui and rest packages. It sends // incoming requests to the correct handler function @@ -54,16 +50,14 @@ func Initialize( cfg config.WebConfig, shutdownChan chan bool, mm message.Manager, - ds storage.Store, mh *msghub.Hub) { webConfig = cfg globalShutdown = shutdownChan // NewContext() will use this DataStore for the web handlers - DataStore = ds msgHub = mh - msgSvc = mm + manager = mm // Content Paths log.Infof("HTTP templates mapped to %q", cfg.TemplateDir) diff --git a/pkg/webui/mailbox_controller.go b/pkg/webui/mailbox_controller.go index ed092e7..16aa305 100644 --- a/pkg/webui/mailbox_controller.go +++ b/pkg/webui/mailbox_controller.go @@ -72,7 +72,7 @@ func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - messages, err := ctx.MsgSvc.GetMetadata(name) + messages, err := ctx.Manager.GetMetadata(name) if err != nil { // This doesn't indicate empty, likely an IO error return fmt.Errorf("Failed to get messages for %v: %v", name, err) @@ -94,7 +94,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - msg, err := ctx.MsgSvc.GetMessage(name, id) + msg, err := ctx.Manager.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -135,7 +135,7 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er if err != nil { return err } - msg, err := ctx.MsgSvc.GetMessage(name, id) + msg, err := ctx.Manager.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -163,7 +163,7 @@ func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( if err != nil { return err } - r, err := ctx.MsgSvc.SourceReader(name, id) + r, err := ctx.Manager.SourceReader(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -198,7 +198,7 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Co http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - msg, err := ctx.MsgSvc.GetMessage(name, id) + msg, err := ctx.Manager.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil @@ -241,7 +241,7 @@ func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Contex http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - msg, err := ctx.MsgSvc.GetMessage(name, id) + msg, err := ctx.Manager.GetMessage(name, id) if err == storage.ErrNotExist { http.NotFound(w, req) return nil From 9be4eec31c20bdcc22995c8cde0074fe475e86de Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Mon, 12 Mar 2018 21:13:57 -0700 Subject: [PATCH 10/26] storage: eliminate ReadBody, ReadRaw for #69 --- pkg/storage/file/fmessage.go | 61 +++++++-------------------------- pkg/storage/file/fstore_test.go | 2 +- pkg/storage/storage.go | 4 --- 3 files changed, 14 insertions(+), 53 deletions(-) diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index e9df65d..ebe8540 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -4,7 +4,6 @@ import ( "bufio" "fmt" "io" - "io/ioutil" "net/mail" "os" "path/filepath" @@ -99,26 +98,6 @@ func (m *Message) rawPath() string { return filepath.Join(m.mailbox.path, m.Fid+".raw") } -// ReadBody opens the .raw portion of a Message and returns a MIMEBody object -func (m *Message) ReadBody() (body *enmime.Envelope, err error) { - file, err := os.Open(m.rawPath()) - if err != nil { - return nil, err - } - defer func() { - if err := file.Close(); err != nil { - log.Errorf("Failed to close %q: %v", m.rawPath(), err) - } - }() - - reader := bufio.NewReader(file) - mime, err := enmime.ReadEnvelope(reader) - if err != nil { - return nil, err - } - return mime, nil -} - // RawReader opens the .raw portion of a Message as an io.ReadCloser func (m *Message) RawReader() (reader io.ReadCloser, err error) { file, err := os.Open(m.rawPath()) @@ -128,26 +107,6 @@ func (m *Message) RawReader() (reader io.ReadCloser, err error) { return file, nil } -// ReadRaw opens the .raw portion of a Message and returns it as a string -func (m *Message) ReadRaw() (raw *string, err error) { - reader, err := m.RawReader() - if err != nil { - return nil, err - } - defer func() { - if err := reader.Close(); err != nil { - log.Errorf("Failed to close %q: %v", m.rawPath(), err) - } - }() - - bodyBytes, err := ioutil.ReadAll(bufio.NewReader(reader)) - if err != nil { - return nil, err - } - bodyString := string(bodyBytes) - return &bodyString, nil -} - // Append data to a newly opened Message, this will fail on a pre-existing Message and // after Close() is called. func (m *Message) Append(data []byte) error { @@ -195,26 +154,32 @@ func (m *Message) Close() error { } } - // Fetch headers - body, err := m.ReadBody() + // Fetch envelope. + // TODO should happen outside of datastore. + r, err := m.RawReader() + if err != nil { + return err + } + env, err := enmime.ReadEnvelope(r) + _ = r.Close() if err != nil { return err } // Only public fields are stored in gob, hence starting with capital F // Parse From address - if address, err := mail.ParseAddress(body.GetHeader("From")); err == nil { + if address, err := mail.ParseAddress(env.GetHeader("From")); err == nil { m.Ffrom = address } else { - m.Ffrom = &mail.Address{Address: body.GetHeader("From")} + m.Ffrom = &mail.Address{Address: env.GetHeader("From")} } - m.Fsubject = body.GetHeader("Subject") + m.Fsubject = env.GetHeader("Subject") // Turn the To header into a slice - if addresses, err := body.AddressList("To"); err == nil { + if addresses, err := env.AddressList("To"); err == nil { m.Fto = addresses } else { - m.Fto = []*mail.Address{{Address: body.GetHeader("To")}} + m.Fto = []*mail.Address{{Address: env.GetHeader("To")}} } // Refresh the index before adding our message diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index eb5add1..70aa168 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -337,7 +337,7 @@ func TestFSMissing(t *testing.T) { assert.Nil(t, err) // Try to read parts of message - _, err = msg.ReadBody() + _, err = msg.RawReader() assert.Error(t, err) if t.Failed() { diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 52e50cc..04f237b 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -7,8 +7,6 @@ import ( "net/mail" "sync" "time" - - "github.com/jhillyerd/enmime" ) var ( @@ -41,8 +39,6 @@ type StoreMessage interface { Date() time.Time Subject() string RawReader() (reader io.ReadCloser, err error) - ReadBody() (body *enmime.Envelope, err error) - ReadRaw() (raw *string, err error) Append(data []byte) error Close() error String() string From 2cc0da3093a56f5265133a0d75d6be1b95962452 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Tue, 13 Mar 2018 22:00:44 -0700 Subject: [PATCH 11/26] storage: More refactoring for #69 - impl Store.AddMessage - file: Use AddMessage() in tests - smtp: Switch to AddMessage - storage: Remove NewMessage, Append, Close methods --- pkg/message/message.go | 51 ++++++++++++++++++ pkg/server/smtp/handler.go | 83 ++++++++++++++++------------ pkg/server/smtp/handler_test.go | 29 ++-------- pkg/storage/file/fmessage.go | 96 +-------------------------------- pkg/storage/file/fstore.go | 58 ++++++++++++++++++++ pkg/storage/file/fstore_test.go | 42 +++++++-------- pkg/storage/retention_test.go | 33 +++++------- pkg/storage/storage.go | 5 +- pkg/storage/testing.go | 6 +++ pkg/test/storage.go | 13 ++--- 10 files changed, 208 insertions(+), 208 deletions(-) diff --git a/pkg/message/message.go b/pkg/message/message.go index fd5f25b..3994ca3 100644 --- a/pkg/message/message.go +++ b/pkg/message/message.go @@ -2,10 +2,13 @@ package message import ( + "io" + "io/ioutil" "net/mail" "time" "github.com/jhillyerd/enmime" + "github.com/jhillyerd/inbucket/pkg/storage" ) // Metadata holds information about a message, but not the content. @@ -24,3 +27,51 @@ type Message struct { Metadata Envelope *enmime.Envelope } + +// Delivery is used to add a message to storage. +type Delivery struct { + Meta Metadata + Reader io.Reader +} + +var _ storage.StoreMessage = &Delivery{} + +// Mailbox getter. +func (d *Delivery) Mailbox() string { + return d.Meta.Mailbox +} + +// ID getter. +func (d *Delivery) ID() string { + return d.Meta.ID +} + +// From getter. +func (d *Delivery) From() *mail.Address { + return d.Meta.From +} + +// To getter. +func (d *Delivery) To() []*mail.Address { + return d.Meta.To +} + +// Date getter. +func (d *Delivery) Date() time.Time { + return d.Meta.Date +} + +// Subject getter. +func (d *Delivery) Subject() string { + return d.Meta.Subject +} + +// Size getter. +func (d *Delivery) Size() int64 { + return d.Meta.Size +} + +// RawReader contains the raw content of the message. +func (d *Delivery) RawReader() (io.ReadCloser, error) { + return ioutil.NopCloser(d.Reader), nil +} diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index b1a6eb3..96fa6db 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -12,7 +12,9 @@ import ( "strings" "time" + "github.com/jhillyerd/enmime" "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/stringutil" ) @@ -442,48 +444,61 @@ func (ss *Session) dataHandler() { // deliverMessage creates and populates a new Message for the specified recipient func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) { - msg, err := ss.server.dataStore.NewMessage(r.localPart) - if err != nil { - ss.logError("Failed to create message for %q: %s", r.localPart, err) - return false - } - - // Generate Received header - stamp := time.Now().Format(timeStampFormat) - recd := fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", - ss.remoteDomain, ss.remoteHost, ss.server.domain, r.address, stamp) - if err := msg.Append([]byte(recd)); err != nil { - ss.logError("Failed to write received header for %q: %s", r.localPart, err) - return false - } - - // Append lines from msgBuf - for _, line := range msgBuf { - if err := msg.Append(line); err != nil { - ss.logError("Failed to append to mailbox %v: %v", r.localPart, err) - // Should really cleanup the crap on filesystem - return false - } - } - if err := msg.Close(); err != nil { - ss.logError("Error while closing message for %v: %v", r.localPart, err) - return false - } name, err := stringutil.ParseMailboxName(r.localPart) if err != nil { // This parse already succeeded when MailboxFor was called, shouldn't fail here. return false } - + buf := bytes.Buffer{} + // Generate Received header + stamp := time.Now().Format(timeStampFormat) + recd := fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", + ss.remoteDomain, ss.remoteHost, ss.server.domain, r.address, stamp) + buf.WriteString(recd) + // Append lines from msgBuf + for _, line := range msgBuf { + buf.Write(line) + } + // TODO replace with something that only reads header? + env, err := enmime.ReadEnvelope(bytes.NewReader(buf.Bytes())) + if err != nil { + ss.logError("Failed to parse message for %q: %v", r.localPart, err) + return false + } + from, err := env.AddressList("From") + if err != nil { + ss.logError("Failed to get From address: %v", err) + return false + } + to, err := env.AddressList("To") + if err != nil { + ss.logError("Failed to get To addresses: %v", err) + return false + } + delivery := &message.Delivery{ + Meta: message.Metadata{ + Mailbox: name, + From: from[0], + To: to, + Date: time.Now(), + Subject: env.GetHeader("Subject"), + }, + Reader: bytes.NewReader(buf.Bytes()), + } + id, err := ss.server.dataStore.AddMessage(delivery) + if err != nil { + ss.logError("Failed to store message for %q: %s", r.localPart, err) + return false + } // Broadcast message information broadcast := msghub.Message{ Mailbox: name, - ID: msg.ID(), - From: msg.From().String(), - To: stringutil.StringAddressList(msg.To()), - Subject: msg.Subject(), - Date: msg.Date(), - Size: msg.Size(), + ID: id, + From: delivery.From().String(), + To: stringutil.StringAddressList(delivery.To()), + Subject: delivery.Subject(), + Date: delivery.Date(), + Size: delivery.Size(), } ss.server.msgHub.Dispatch(broadcast) diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index ca9ff2e..94b0d48 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -8,7 +8,6 @@ import ( "log" "net" - "net/mail" "net/textproto" "os" "testing" @@ -141,18 +140,7 @@ func TestReadyState(t *testing.T) { // Test commands in MAIL state func TestMailState(t *testing.T) { - // Setup mock objects - mds := &storage.MockDataStore{} - msg1 := &storage.MockMessage{} - mds.On("NewMessage", "u1").Return(msg1, nil) - msg1.On("ID").Return("") - msg1.On("From").Return(&mail.Address{}) - msg1.On("To").Return(make([]*mail.Address, 0)) - msg1.On("Date").Return(time.Time{}) - msg1.On("Subject").Return("") - msg1.On("Size").Return(0) - msg1.On("Close").Return(nil) - + mds := test.NewStore() server, logbuf, teardown := setupSMTPServer(mds) defer teardown() @@ -214,7 +202,7 @@ func TestMailState(t *testing.T) { {"MAIL FROM:", 250}, {"RCPT TO:", 250}, {"DATA", 354}, - {".", 250}, + {".", 451}, } if err := playSession(t, server, script); err != nil { t.Error(err) @@ -253,18 +241,7 @@ func TestMailState(t *testing.T) { // Test commands in DATA state func TestDataState(t *testing.T) { - // Setup mock objects - mds := &storage.MockDataStore{} - msg1 := &storage.MockMessage{} - mds.On("NewMessage", "u1").Return(msg1, nil) - msg1.On("ID").Return("") - msg1.On("From").Return(&mail.Address{}) - msg1.On("To").Return(make([]*mail.Address, 0)) - msg1.On("Date").Return(time.Time{}) - msg1.On("Subject").Return("") - msg1.On("Size").Return(0) - msg1.On("Close").Return(nil) - + mds := test.NewStore() server, logbuf, teardown := setupSMTPServer(mds) defer teardown() diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index ebe8540..cf06d5c 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -2,16 +2,13 @@ package file import ( "bufio" - "fmt" "io" "net/mail" "os" "path/filepath" "time" - "github.com/jhillyerd/enmime" "github.com/jhillyerd/inbucket/pkg/log" - "github.com/jhillyerd/inbucket/pkg/storage" ) // Message implements Message and contains a little bit of data about a @@ -33,7 +30,7 @@ type Message struct { // newMessage creates a new FileMessage object and sets the Date and ID fields. // It will also delete messages over messageCap if configured. -func (mb *mbox) newMessage() (storage.StoreMessage, error) { +func (mb *mbox) newMessage() (*Message, error) { // Load index if !mb.indexLoaded { if err := mb.readIndex(); err != nil { @@ -84,11 +81,6 @@ func (m *Message) Subject() string { return m.Fsubject } -// String returns a string in the form: "Subject()" from From() -func (m *Message) String() string { - return fmt.Sprintf("\"%v\" from %v", m.Fsubject, m.Ffrom) -} - // Size returns the size of the Message on disk in bytes func (m *Message) Size() int64 { return m.Fsize @@ -106,89 +98,3 @@ func (m *Message) RawReader() (reader io.ReadCloser, err error) { } return file, nil } - -// Append data to a newly opened Message, this will fail on a pre-existing Message and -// after Close() is called. -func (m *Message) Append(data []byte) error { - // Prevent Appending to a pre-existing Message - if !m.writable { - return storage.ErrNotWritable - } - // Open file for writing if we haven't yet - if m.writer == nil { - // Ensure mailbox directory exists - if err := m.mailbox.createDir(); err != nil { - return err - } - file, err := os.Create(m.rawPath()) - if err != nil { - // Set writable false just in case something calls me a million times - m.writable = false - return err - } - m.writerFile = file - m.writer = bufio.NewWriter(file) - } - _, err := m.writer.Write(data) - m.Fsize += int64(len(data)) - return err -} - -// Close this Message for writing - no more data may be Appended. Close() will also -// trigger the creation of the .gob file. -func (m *Message) Close() error { - // nil out the writer fields so they can't be used - writer := m.writer - writerFile := m.writerFile - m.writer = nil - m.writerFile = nil - - if writer != nil { - if err := writer.Flush(); err != nil { - return err - } - } - if writerFile != nil { - if err := writerFile.Close(); err != nil { - return err - } - } - - // Fetch envelope. - // TODO should happen outside of datastore. - r, err := m.RawReader() - if err != nil { - return err - } - env, err := enmime.ReadEnvelope(r) - _ = r.Close() - if err != nil { - return err - } - - // Only public fields are stored in gob, hence starting with capital F - // Parse From address - if address, err := mail.ParseAddress(env.GetHeader("From")); err == nil { - m.Ffrom = address - } else { - m.Ffrom = &mail.Address{Address: env.GetHeader("From")} - } - m.Fsubject = env.GetHeader("Subject") - - // Turn the To header into a slice - if addresses, err := env.AddressList("To"); err == nil { - m.Fto = addresses - } else { - m.Fto = []*mail.Address{{Address: env.GetHeader("To")}} - } - - // Refresh the index before adding our message - err = m.mailbox.readIndex() - if err != nil { - return err - } - - // Made it this far without errors, add it to the index - m.mailbox.messages = append(m.mailbox.messages, m) - return m.mailbox.writeIndex() -} diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index 3e5f9bb..c9ecfee 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -74,6 +74,64 @@ func New(cfg config.DataStoreConfig) storage.Store { return &Store{path: path, mailPath: mailPath, messageCap: cfg.MailboxMsgCap} } +// AddMessage adds a message to the specified mailbox. +func (fs *Store) AddMessage(m storage.StoreMessage) (id string, err error) { + r, err := m.RawReader() + if err != nil { + return "", err + } + mb, err := fs.mbox(m.Mailbox()) + if err != nil { + return "", err + } + // Create a new message. + fm, err := mb.newMessage() + if err != nil { + return "", err + } + // Ensure mailbox directory exists. + if err := mb.createDir(); err != nil { + return "", err + } + // Write the message content + file, err := os.Create(fm.rawPath()) + if err != nil { + return "", err + } + w := bufio.NewWriter(file) + size, err := io.Copy(w, r) + if err != nil { + // Try to remove the file + _ = file.Close() + _ = os.Remove(fm.rawPath()) + return "", err + } + _ = r.Close() + if err := w.Flush(); err != nil { + // Try to remove the file + _ = file.Close() + _ = os.Remove(fm.rawPath()) + return "", err + } + if err := file.Close(); err != nil { + // Try to remove the file + _ = os.Remove(fm.rawPath()) + return "", err + } + // Update the index. + fm.Fdate = m.Date() + fm.Ffrom = m.From() + fm.Fsize = size + fm.Fsubject = m.Subject() + mb.messages = append(mb.messages, fm) + if err := mb.writeIndex(); err != nil { + // Try to remove the file + _ = os.Remove(fm.rawPath()) + return "", err + } + return fm.Fid, nil +} + // GetMessage returns the messages in the named mailbox, or an error. func (fs *Store) GetMessage(mailbox, id string) (storage.StoreMessage, error) { mb, err := fs.mbox(mailbox) diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 70aa168..4b2f287 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -6,12 +6,15 @@ import ( "io" "io/ioutil" "log" + "net/mail" "os" "path/filepath" + "strings" "testing" "time" "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/stretchr/testify/assert" ) @@ -480,32 +483,25 @@ func setupDataStore(cfg config.DataStoreConfig) (*Store, *bytes.Buffer) { // deliverMessage creates and delivers a message to the specific mailbox, returning // the size of the generated message. -func deliverMessage(ds *Store, mbName string, subject string, - date time.Time) (id string, size int64) { - // Build fake SMTP message for delivery - testMsg := make([]byte, 0, 300) - testMsg = append(testMsg, []byte("To: somebody@host\r\n")...) - testMsg = append(testMsg, []byte("From: somebodyelse@host\r\n")...) - testMsg = append(testMsg, []byte(fmt.Sprintf("Subject: %s\r\n", subject))...) - testMsg = append(testMsg, []byte("\r\n")...) - testMsg = append(testMsg, []byte("Test Body\r\n")...) - - // Create message object - id = generateID(date) - msg, err := ds.NewMessage(mbName) +func deliverMessage(ds *Store, mbName string, subject string, date time.Time) (string, int64) { + // Build message for delivery + meta := message.Metadata{ + Mailbox: mbName, + To: []*mail.Address{{Name: "", Address: "somebody@host"}}, + From: &mail.Address{Name: "", Address: "somebodyelse@host"}, + Subject: subject, + Date: date, + } + testMsg := fmt.Sprintf("To: %s\r\nFrom: %s\r\nSubject: %s\r\n\r\nTest Body\r\n", + meta.To[0].Address, meta.From.Address, subject) + delivery := &message.Delivery{ + Meta: meta, + Reader: ioutil.NopCloser(strings.NewReader(testMsg)), + } + id, err := ds.AddMessage(delivery) if err != nil { panic(err) } - fmsg := msg.(*Message) - fmsg.Fdate = date - fmsg.Fid = id - if err = msg.Append(testMsg); err != nil { - panic(err) - } - if err = msg.Close(); err != nil { - panic(err) - } - return id, int64(len(testMsg)) } diff --git a/pkg/storage/retention_test.go b/pkg/storage/retention_test.go index f6ed828..5a0d07e 100644 --- a/pkg/storage/retention_test.go +++ b/pkg/storage/retention_test.go @@ -13,24 +13,18 @@ import ( func TestDoRetentionScan(t *testing.T) { ds := test.NewStore() // Mockup some different aged messages (num is in hours) - new1 := mockMessage(0) - new2 := mockMessage(1) - new3 := mockMessage(2) - old1 := mockMessage(4) - old2 := mockMessage(12) - old3 := mockMessage(24) - ds.AddMessage("mb1", new1) - new1.On("Mailbox").Return("mb1") - ds.AddMessage("mb1", old1) - old1.On("Mailbox").Return("mb1") - ds.AddMessage("mb1", old2) - old2.On("Mailbox").Return("mb1") - ds.AddMessage("mb2", old3) - old3.On("Mailbox").Return("mb2") - ds.AddMessage("mb2", new2) - new2.On("Mailbox").Return("mb2") - ds.AddMessage("mb3", new3) - new3.On("Mailbox").Return("mb3") + new1 := mockMessage("mb1", 0) + new2 := mockMessage("mb2", 1) + new3 := mockMessage("mb3", 2) + old1 := mockMessage("mb1", 4) + old2 := mockMessage("mb1", 12) + old3 := mockMessage("mb2", 24) + ds.AddMessage(new1) + ds.AddMessage(old1) + ds.AddMessage(old2) + ds.AddMessage(old3) + ds.AddMessage(new2) + ds.AddMessage(new3) // Test 4 hour retention cfg := config.DataStoreConfig{ RetentionMinutes: 239, @@ -56,8 +50,9 @@ func TestDoRetentionScan(t *testing.T) { } // Make a MockMessage of a specific age -func mockMessage(ageHours int) *storage.MockMessage { +func mockMessage(mailbox string, ageHours int) *storage.MockMessage { msg := &storage.MockMessage{} + msg.On("Mailbox").Return(mailbox) msg.On("ID").Return(fmt.Sprintf("MSG[age=%vh]", ageHours)) msg.On("Date").Return(time.Now().Add(time.Duration(ageHours*-1) * time.Hour)) msg.On("Delete").Return(nil) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 04f237b..8f6783a 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -19,6 +19,8 @@ var ( // Store is the interface Inbucket uses to interact with storage implementations. type Store interface { + // AddMessage stores the message, message ID and Size will be ignored. + AddMessage(message StoreMessage) (id string, err error) GetMessage(mailbox, id string) (StoreMessage, error) GetMessages(mailbox string) ([]StoreMessage, error) PurgeMessages(mailbox string) error @@ -39,8 +41,5 @@ type StoreMessage interface { Date() time.Time Subject() string RawReader() (reader io.ReadCloser, err error) - Append(data []byte) error - Close() error - String() string Size() int64 } diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go index 9defa55..b987477 100644 --- a/pkg/storage/testing.go +++ b/pkg/storage/testing.go @@ -15,6 +15,12 @@ type MockDataStore struct { mock.Mock } +// AddMessage mock function +func (m *MockDataStore) AddMessage(message StoreMessage) (string, error) { + args := m.Called(message) + return args.String(0), args.Error(1) +} + // GetMessage mock function func (m *MockDataStore) GetMessage(name, id string) (StoreMessage, error) { args := m.Called(name, id) diff --git a/pkg/test/storage.go b/pkg/test/storage.go index 92195ff..52c9b00 100644 --- a/pkg/test/storage.go +++ b/pkg/test/storage.go @@ -23,9 +23,11 @@ func NewStore() *StoreStub { } // AddMessage adds a message to the specified mailbox. -func (s *StoreStub) AddMessage(mailbox string, m storage.StoreMessage) { - msgs := s.mailboxes[mailbox] - s.mailboxes[mailbox] = append(msgs, m) +func (s *StoreStub) AddMessage(m storage.StoreMessage) (id string, err error) { + mb := m.Mailbox() + msgs := s.mailboxes[mb] + s.mailboxes[mb] = append(msgs, m) + return m.ID(), nil } // GetMessage gets a message by ID from the specified mailbox. @@ -80,11 +82,6 @@ func (s *StoreStub) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) e return nil } -// NewMessage is temproary until #69 MessageData refactor -func (s *StoreStub) NewMessage(mailbox string) (storage.StoreMessage, error) { - return nil, nil -} - // LockFor mock function returns a new RWMutex, never errors. // TODO(#69) remove func (s *StoreStub) LockFor(name string) (*sync.RWMutex, error) { From 519779b7baa45a5cc5e094c9c94a1500e4ab2982 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Wed, 14 Mar 2018 21:05:59 -0700 Subject: [PATCH 12/26] storage: eliminate mocks, closes #80 --- pkg/rest/testutils_test.go | 27 ------ pkg/storage/retention_test.go | 30 +++---- pkg/storage/testing.go | 152 ---------------------------------- 3 files changed, 16 insertions(+), 193 deletions(-) delete mode 100644 pkg/storage/testing.go diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index cfddede..23996a5 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -9,12 +9,10 @@ import ( "net/mail" "time" - "github.com/jhillyerd/enmime" "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/server/web" - "github.com/jhillyerd/inbucket/pkg/storage" ) type InputMessageData struct { @@ -26,31 +24,6 @@ type InputMessageData struct { HTML, Text string } -func (d *InputMessageData) MockMessage() *storage.MockMessage { - from, _ := mail.ParseAddress(d.From) - to := make([]*mail.Address, len(d.To)) - for i, a := range d.To { - to[i], _ = mail.ParseAddress(a) - } - msg := &storage.MockMessage{} - msg.On("ID").Return(d.ID) - msg.On("From").Return(from) - msg.On("To").Return(to) - msg.On("Subject").Return(d.Subject) - msg.On("Date").Return(d.Date) - msg.On("Size").Return(d.Size) - gomsg := &mail.Message{ - Header: d.Header, - } - msg.On("ReadHeader").Return(gomsg, nil) - body := &enmime.Envelope{ - Text: d.Text, - HTML: d.HTML, - } - msg.On("ReadBody").Return(body, nil) - return msg -} - // isJSONStringEqual is a utility function to return a nicely formatted message when // comparing a string to a value received from a JSON map. func isJSONStringEqual(key, expected string, received interface{}) (message string, ok bool) { diff --git a/pkg/storage/retention_test.go b/pkg/storage/retention_test.go index 5a0d07e..a49cc59 100644 --- a/pkg/storage/retention_test.go +++ b/pkg/storage/retention_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/test" ) @@ -13,12 +14,12 @@ import ( func TestDoRetentionScan(t *testing.T) { ds := test.NewStore() // Mockup some different aged messages (num is in hours) - new1 := mockMessage("mb1", 0) - new2 := mockMessage("mb2", 1) - new3 := mockMessage("mb3", 2) - old1 := mockMessage("mb1", 4) - old2 := mockMessage("mb1", 12) - old3 := mockMessage("mb2", 24) + new1 := stubMessage("mb1", 0) + new2 := stubMessage("mb2", 1) + new3 := stubMessage("mb3", 2) + old1 := stubMessage("mb1", 4) + old2 := stubMessage("mb1", 12) + old3 := stubMessage("mb2", 24) ds.AddMessage(new1) ds.AddMessage(old1) ds.AddMessage(old2) @@ -49,12 +50,13 @@ func TestDoRetentionScan(t *testing.T) { } } -// Make a MockMessage of a specific age -func mockMessage(mailbox string, ageHours int) *storage.MockMessage { - msg := &storage.MockMessage{} - msg.On("Mailbox").Return(mailbox) - msg.On("ID").Return(fmt.Sprintf("MSG[age=%vh]", ageHours)) - msg.On("Date").Return(time.Now().Add(time.Duration(ageHours*-1) * time.Hour)) - msg.On("Delete").Return(nil) - return msg +// stubMessage creates a message stub of a specific age +func stubMessage(mailbox string, ageHours int) storage.StoreMessage { + return &message.Delivery{ + Meta: message.Metadata{ + Mailbox: mailbox, + ID: fmt.Sprintf("MSG[age=%vh]", ageHours), + Date: time.Now().Add(time.Duration(ageHours*-1) * time.Hour), + }, + } } diff --git a/pkg/storage/testing.go b/pkg/storage/testing.go deleted file mode 100644 index b987477..0000000 --- a/pkg/storage/testing.go +++ /dev/null @@ -1,152 +0,0 @@ -package storage - -import ( - "io" - "net/mail" - "sync" - "time" - - "github.com/jhillyerd/enmime" - "github.com/stretchr/testify/mock" -) - -// MockDataStore is a shared mock for unit testing -type MockDataStore struct { - mock.Mock -} - -// AddMessage mock function -func (m *MockDataStore) AddMessage(message StoreMessage) (string, error) { - args := m.Called(message) - return args.String(0), args.Error(1) -} - -// GetMessage mock function -func (m *MockDataStore) GetMessage(name, id string) (StoreMessage, error) { - args := m.Called(name, id) - return args.Get(0).(StoreMessage), args.Error(1) -} - -// GetMessages mock function -func (m *MockDataStore) GetMessages(name string) ([]StoreMessage, error) { - args := m.Called(name) - return args.Get(0).([]StoreMessage), args.Error(1) -} - -// RemoveMessage mock function -func (m *MockDataStore) RemoveMessage(name, id string) error { - args := m.Called(name, id) - return args.Error(0) -} - -// PurgeMessages mock function -func (m *MockDataStore) PurgeMessages(name string) error { - args := m.Called(name) - return args.Error(0) -} - -// LockFor mock function returns a new RWMutex, never errors. -func (m *MockDataStore) LockFor(name string) (*sync.RWMutex, error) { - return &sync.RWMutex{}, nil -} - -// NewMessage temporary for #69 -func (m *MockDataStore) NewMessage(mailbox string) (StoreMessage, error) { - args := m.Called(mailbox) - return args.Get(0).(StoreMessage), args.Error(1) -} - -// VisitMailboxes accepts a function that will be called with the messages in each mailbox while it -// continues to return true. -func (m *MockDataStore) VisitMailboxes(f func([]StoreMessage) (cont bool)) error { - return nil -} - -// MockMessage is a shared mock for unit testing -type MockMessage struct { - mock.Mock -} - -// Mailbox mock function -func (m *MockMessage) Mailbox() string { - args := m.Called() - return args.String(0) -} - -// ID mock function -func (m *MockMessage) ID() string { - args := m.Called() - return args.String(0) -} - -// From mock function -func (m *MockMessage) From() *mail.Address { - args := m.Called() - return args.Get(0).(*mail.Address) -} - -// To mock function -func (m *MockMessage) To() []*mail.Address { - args := m.Called() - return args.Get(0).([]*mail.Address) -} - -// Date mock function -func (m *MockMessage) Date() time.Time { - args := m.Called() - return args.Get(0).(time.Time) -} - -// Subject mock function -func (m *MockMessage) Subject() string { - args := m.Called() - return args.String(0) -} - -// ReadHeader mock function -func (m *MockMessage) ReadHeader() (msg *mail.Message, err error) { - args := m.Called() - return args.Get(0).(*mail.Message), args.Error(1) -} - -// ReadBody mock function -func (m *MockMessage) ReadBody() (body *enmime.Envelope, err error) { - args := m.Called() - return args.Get(0).(*enmime.Envelope), args.Error(1) -} - -// ReadRaw mock function -func (m *MockMessage) ReadRaw() (raw *string, err error) { - args := m.Called() - return args.Get(0).(*string), args.Error(1) -} - -// RawReader mock function -func (m *MockMessage) RawReader() (reader io.ReadCloser, err error) { - args := m.Called() - return args.Get(0).(io.ReadCloser), args.Error(1) -} - -// Size mock function -func (m *MockMessage) Size() int64 { - args := m.Called() - return int64(args.Int(0)) -} - -// Append mock function -func (m *MockMessage) Append(data []byte) error { - // []byte arg seems to mess up testify/mock - return nil -} - -// Close mock function -func (m *MockMessage) Close() error { - args := m.Called() - return args.Error(0) -} - -// String mock function -func (m *MockMessage) String() string { - args := m.Called() - return args.String(0) -} From 5e13e5076301a9b2eaaaa95a8e4b35df93d39b0f Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Wed, 14 Mar 2018 22:51:40 -0700 Subject: [PATCH 13/26] test: Start work on test suite for #82 - smtp: Tidy up []byte/buffer/string use in delivery #69 --- pkg/server/smtp/handler.go | 60 ++++++++-------------- pkg/storage/file/fstore.go | 1 + pkg/storage/file/fstore_test.go | 14 ++++++ pkg/storage/storage.go | 2 - pkg/test/storage_suite.go | 89 +++++++++++++++++++++++++++++++++ 5 files changed, 124 insertions(+), 42 deletions(-) create mode 100644 pkg/test/storage_suite.go diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 96fa6db..d9dad39 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -352,7 +352,6 @@ func (ss *Session) mailHandler(cmd string, arg string) { func (ss *Session) dataHandler() { recipients := make([]recipientDetails, 0, ss.recipients.Len()) // Get a Mailbox and a new Message for each recipient - msgSize := 0 if ss.server.storeMessages { for e := ss.recipients.Front(); e != nil; e = e.Next() { recip := e.Value.(string) @@ -373,11 +372,9 @@ func (ss *Session) dataHandler() { } ss.send("354 Start mail input; end with .") - var lineBuf bytes.Buffer - msgBuf := make([][]byte, 0, 1024) + msgBuf := &bytes.Buffer{} for { - lineBuf.Reset() - err := ss.readByteLine(&lineBuf) + lineBuf, err := ss.readByteLine() if err != nil { if netErr, ok := err.(net.Error); ok { if netErr.Timeout() { @@ -388,9 +385,7 @@ func (ss *Session) dataHandler() { ss.enterState(QUIT) return } - line := lineBuf.Bytes() - // ss.logTrace("DATA: %q", line) - if string(line) == ".\r\n" || string(line) == ".\n" { + if bytes.Equal(lineBuf, []byte(".\r\n")) || bytes.Equal(lineBuf, []byte(".\n")) { // Mail data complete if ss.server.storeMessages { // Create a message for each valid recipient @@ -405,7 +400,7 @@ func (ss *Session) dataHandler() { return } mu.Lock() - ok := ss.deliverMessage(r, msgBuf) + ok := ss.deliverMessage(r, msgBuf.Bytes()) mu.Unlock() if ok { expReceivedTotal.Add(1) @@ -420,47 +415,34 @@ func (ss *Session) dataHandler() { expReceivedTotal.Add(1) } ss.send("250 Mail accepted for delivery") - ss.logInfo("Message size %v bytes", msgSize) + ss.logInfo("Message size %v bytes", msgBuf.Len()) ss.reset() return } // SMTP RFC says remove leading periods from input - if len(line) > 0 && line[0] == '.' { - line = line[1:] + if len(lineBuf) > 0 && lineBuf[0] == '.' { + lineBuf = lineBuf[1:] } - // Second append copies line/lineBuf so we can reuse it - msgBuf = append(msgBuf, append([]byte{}, line...)) - msgSize += len(line) - if msgSize > ss.server.maxMessageBytes { + msgBuf.Write(lineBuf) + if msgBuf.Len() > ss.server.maxMessageBytes { // Max message size exceeded ss.send("552 Maximum message size exceeded") ss.logWarn("Max message size exceeded while in DATA") ss.reset() - // Should really cleanup the crap on filesystem (after issue #23) return } } // end for } // deliverMessage creates and populates a new Message for the specified recipient -func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) { +func (ss *Session) deliverMessage(r recipientDetails, content []byte) (ok bool) { name, err := stringutil.ParseMailboxName(r.localPart) if err != nil { // This parse already succeeded when MailboxFor was called, shouldn't fail here. return false } - buf := bytes.Buffer{} - // Generate Received header - stamp := time.Now().Format(timeStampFormat) - recd := fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", - ss.remoteDomain, ss.remoteHost, ss.server.domain, r.address, stamp) - buf.WriteString(recd) - // Append lines from msgBuf - for _, line := range msgBuf { - buf.Write(line) - } // TODO replace with something that only reads header? - env, err := enmime.ReadEnvelope(bytes.NewReader(buf.Bytes())) + env, err := enmime.ReadEnvelope(bytes.NewReader(content)) if err != nil { ss.logError("Failed to parse message for %q: %v", r.localPart, err) return false @@ -475,6 +457,10 @@ func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) ss.logError("Failed to get To addresses: %v", err) return false } + // Generate Received header. + stamp := time.Now().Format(timeStampFormat) + recd := strings.NewReader(fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", + ss.remoteDomain, ss.remoteHost, ss.server.domain, r.address, stamp)) delivery := &message.Delivery{ Meta: message.Metadata{ Mailbox: name, @@ -483,14 +469,14 @@ func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) Date: time.Now(), Subject: env.GetHeader("Subject"), }, - Reader: bytes.NewReader(buf.Bytes()), + Reader: io.MultiReader(recd, bytes.NewReader(content)), } id, err := ss.server.dataStore.AddMessage(delivery) if err != nil { ss.logError("Failed to store message for %q: %s", r.localPart, err) return false } - // Broadcast message information + // Broadcast message information. broadcast := msghub.Message{ Mailbox: name, ID: id, @@ -501,7 +487,6 @@ func (ss *Session) deliverMessage(r recipientDetails, msgBuf [][]byte) (ok bool) Size: delivery.Size(), } ss.server.msgHub.Dispatch(broadcast) - return true } @@ -535,16 +520,11 @@ func (ss *Session) send(msg string) { // readByteLine reads a line of input into the provided buffer. Does // not reset the Buffer - please do so prior to calling. -func (ss *Session) readByteLine(buf io.Writer) error { +func (ss *Session) readByteLine() ([]byte, error) { if err := ss.conn.SetReadDeadline(ss.nextDeadline()); err != nil { - return err + return nil, err } - line, err := ss.reader.ReadBytes('\n') - if err != nil { - return err - } - _, err = buf.Write(line) - return err + return ss.reader.ReadBytes('\n') } // Reads a line of input diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index c9ecfee..b41b5d3 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -121,6 +121,7 @@ func (fs *Store) AddMessage(m storage.StoreMessage) (id string, err error) { // Update the index. fm.Fdate = m.Date() fm.Ffrom = m.From() + fm.Fto = m.To() fm.Fsize = size fm.Fsubject = m.Subject() mb.messages = append(mb.messages, fm) diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 4b2f287..94380d9 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -16,9 +16,23 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/test" "github.com/stretchr/testify/assert" ) +// TestSuite runs storage package test suite on file store. +func TestSuite(t *testing.T) { + ds, logbuf := setupDataStore(config.DataStoreConfig{}) + defer teardownDataStore(ds) + test.StoreSuite(t, ds) + if t.Failed() { + // Wait for handler to finish logging. + time.Sleep(2 * time.Second) + // Dump buffered log data if there was a failure. + _, _ = io.Copy(os.Stderr, logbuf) + } +} + // Test directory structure created by filestore func TestFSDirStructure(t *testing.T) { ds, logbuf := setupDataStore(config.DataStoreConfig{}) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 8f6783a..62d4972 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -28,8 +28,6 @@ type Store interface { VisitMailboxes(f func([]StoreMessage) (cont bool)) error // LockFor is a temporary hack to fix #77 until Datastore revamp LockFor(emailAddress string) (*sync.RWMutex, error) - // NewMessage is temproary until #69 MessageData refactor - NewMessage(mailbox string) (StoreMessage, error) } // StoreMessage represents a message to be stored, or returned from a storage implementation. diff --git a/pkg/test/storage_suite.go b/pkg/test/storage_suite.go new file mode 100644 index 0000000..fd71774 --- /dev/null +++ b/pkg/test/storage_suite.go @@ -0,0 +1,89 @@ +package test + +import ( + "net/mail" + "strings" + "testing" + "time" + + "github.com/jhillyerd/inbucket/pkg/message" + "github.com/jhillyerd/inbucket/pkg/storage" +) + +// StoreSuite runs a set of general tests on the provided Store +func StoreSuite(t *testing.T, store storage.Store) { + testCases := []struct { + name string + test func(*testing.T, storage.Store) + }{ + {"metadata", testMetadata}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.test(t, store) + }) + } +} + +func testMetadata(t *testing.T, ds storage.Store) { + // Store a message + mailbox := "testmailbox" + from := &mail.Address{Name: "From Person", Address: "from@person.com"} + to := []*mail.Address{ + {Name: "One Person", Address: "one@a.person.com"}, + {Name: "Two Person", Address: "two@b.person.com"}, + } + date := time.Now() + subject := "fantastic test subject line" + content := "doesn't matter" + delivery := &message.Delivery{ + Meta: message.Metadata{ + // ID and Size will be determined by the Store + Mailbox: mailbox, + From: from, + To: to, + Date: date, + Subject: subject, + }, + Reader: strings.NewReader(content), + } + id, err := ds.AddMessage(delivery) + if err != nil { + t.Fatal(err) + } + if id == "" { + t.Fatal("Expected AddMessage() to return non-empty ID string") + } + // Retrieve and validate the message + sm, err := ds.GetMessage(mailbox, id) + if err != nil { + t.Fatal(err) + } + if sm.Mailbox() != mailbox { + t.Errorf("got mailbox %q, want: %q", sm.Mailbox(), mailbox) + } + if sm.ID() != id { + t.Errorf("got id %q, want: %q", sm.ID(), id) + } + if *sm.From() != *from { + t.Errorf("got from %v, want: %v", sm.From(), from) + } + if len(sm.To()) != len(to) { + t.Errorf("got len(to) = %v, want: %v", len(sm.To()), len(to)) + } else { + for i, got := range sm.To() { + if *to[i] != *got { + t.Errorf("got to[%v] %v, want: %v", i, got, to[i]) + } + } + } + if !sm.Date().Equal(date) { + t.Errorf("got date %v, want: %v", sm.Date(), date) + } + if sm.Subject() != subject { + t.Errorf("got subject %q, want: %q", sm.Subject(), subject) + } + if sm.Size() != int64(len(content)) { + t.Errorf("got size %v, want: %v", sm.Size(), len(content)) + } +} From 9b3d3c2ea821586d669cfbd06f3b346f2531c2d2 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Fri, 16 Mar 2018 22:05:07 -0700 Subject: [PATCH 14/26] test: Finish initial storage test suite, closes #82 --- Makefile | 5 +- pkg/storage/file/fstore_test.go | 235 +---------------------------- pkg/test/storage_suite.go | 257 +++++++++++++++++++++++++++++++- 3 files changed, 260 insertions(+), 237 deletions(-) diff --git a/Makefile b/Makefile index 821c674..eacf4ba 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ SHELL = /bin/sh SRC := $(shell find . -type f -name '*.go' -not -path "./vendor/*") PKGS := $(shell go list ./... | grep -v /vendor/) -.PHONY: all build clean fmt lint simplify test +.PHONY: all build clean fmt lint reflex simplify test commands = client inbucket @@ -34,3 +34,6 @@ lint: @test -z "$(shell gofmt -l . | tee /dev/stderr)" || echo "[WARN] Fix formatting issues with 'make fmt'" @golint -set_exit_status $(PKGS) @go vet $(PKGS) + +reflex: + reflex -r '\.go$$' -- sh -c 'echo; date; echo; go test ./...' diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 94380d9..18b1fe7 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -22,15 +22,13 @@ import ( // TestSuite runs storage package test suite on file store. func TestSuite(t *testing.T) { - ds, logbuf := setupDataStore(config.DataStoreConfig{}) - defer teardownDataStore(ds) - test.StoreSuite(t, ds) - if t.Failed() { - // Wait for handler to finish logging. - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure. - _, _ = io.Copy(os.Stderr, logbuf) - } + test.StoreSuite(t, func() (storage.Store, func(), error) { + ds, _ := setupDataStore(config.DataStoreConfig{}) + destroy := func() { + teardownDataStore(ds) + } + return ds, destroy, nil + }) } // Test directory structure created by filestore @@ -111,225 +109,6 @@ func TestFSDirStructure(t *testing.T) { } } -// TestFSVisitMailboxes tests VisitMailboxes -func TestFSVisitMailboxes(t *testing.T) { - ds, logbuf := setupDataStore(config.DataStoreConfig{}) - defer teardownDataStore(ds) - boxes := []string{"abby", "bill", "christa", "donald", "evelyn"} - for _, name := range boxes { - // Create day old message - date := time.Now().Add(-24 * time.Hour) - deliverMessage(ds, name, "Old Message", date) - - // Create current message - date = time.Now() - deliverMessage(ds, name, "New Message", date) - } - - seen := 0 - err := ds.VisitMailboxes(func(messages []storage.StoreMessage) bool { - seen++ - count := len(messages) - if count != 2 { - t.Errorf("got: %v messages, want: 2", count) - } - return true - }) - assert.Nil(t, err) - assert.Equal(t, 5, seen) - - if t.Failed() { - // Wait for handler to finish logging - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure - _, _ = io.Copy(os.Stderr, logbuf) - } -} - -// Test delivering several messages to the same mailbox, meanwhile querying its -// contents with a new mailbox object each time -func TestFSDeliverMany(t *testing.T) { - ds, logbuf := setupDataStore(config.DataStoreConfig{}) - defer teardownDataStore(ds) - - mbName := "fred" - subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} - - for i, subj := range subjects { - // Check number of messages - msgs, err := ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - assert.Equal(t, i, len(msgs), "Expected %v message(s), but got %v", i, len(msgs)) - - // Add a message - deliverMessage(ds, mbName, subj, time.Now()) - } - - msgs, err := ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - assert.Equal(t, len(subjects), len(msgs), "Expected %v message(s), but got %v", - len(subjects), len(msgs)) - - // Confirm delivery order - for i, expect := range subjects { - subj := msgs[i].Subject() - assert.Equal(t, expect, subj, "Expected subject %q, got %q", expect, subj) - } - - if t.Failed() { - // Wait for handler to finish logging - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure - _, _ = io.Copy(os.Stderr, logbuf) - } -} - -// Test deleting messages -func TestFSDelete(t *testing.T) { - ds, logbuf := setupDataStore(config.DataStoreConfig{}) - defer teardownDataStore(ds) - - mbName := "fred" - subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} - - for _, subj := range subjects { - // Add a message - deliverMessage(ds, mbName, subj, time.Now()) - } - - msgs, err := ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - assert.Equal(t, len(subjects), len(msgs), "Expected %v message(s), but got %v", - len(subjects), len(msgs)) - - // Delete a couple messages - err = ds.RemoveMessage(mbName, msgs[1].ID()) - if err != nil { - t.Fatal(err) - } - err = ds.RemoveMessage(mbName, msgs[3].ID()) - if err != nil { - t.Fatal(err) - } - - // Confirm deletion - msgs, err = ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - - subjects = []string{"alpha", "charlie", "echo"} - assert.Equal(t, len(subjects), len(msgs), "Expected %v message(s), but got %v", - len(subjects), len(msgs)) - for i, expect := range subjects { - subj := msgs[i].Subject() - assert.Equal(t, expect, subj, "Expected subject %q, got %q", expect, subj) - } - - // Try appending one more - deliverMessage(ds, mbName, "foxtrot", time.Now()) - - msgs, err = ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - - subjects = []string{"alpha", "charlie", "echo", "foxtrot"} - assert.Equal(t, len(subjects), len(msgs), "Expected %v message(s), but got %v", - len(subjects), len(msgs)) - for i, expect := range subjects { - subj := msgs[i].Subject() - assert.Equal(t, expect, subj, "Expected subject %q, got %q", expect, subj) - } - - if t.Failed() { - // Wait for handler to finish logging - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure - _, _ = io.Copy(os.Stderr, logbuf) - } -} - -// Test purging a mailbox -func TestFSPurge(t *testing.T) { - ds, logbuf := setupDataStore(config.DataStoreConfig{}) - defer teardownDataStore(ds) - - mbName := "fred" - subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} - - for _, subj := range subjects { - // Add a message - deliverMessage(ds, mbName, subj, time.Now()) - } - - msgs, err := ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - assert.Equal(t, len(subjects), len(msgs), "Expected %v message(s), but got %v", - len(subjects), len(msgs)) - - // Purge mailbox - err = ds.PurgeMessages(mbName) - assert.Nil(t, err) - - // Confirm deletion - msgs, err = ds.GetMessages(mbName) - if err != nil { - t.Fatalf("Failed to GetMessages for %q: %v", mbName, err) - } - - assert.Equal(t, len(msgs), 0, "Expected mailbox to have zero messages, got %v", len(msgs)) - - if t.Failed() { - // Wait for handler to finish logging - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure - _, _ = io.Copy(os.Stderr, logbuf) - } -} - -// Test message size calculation -func TestFSSize(t *testing.T) { - ds, logbuf := setupDataStore(config.DataStoreConfig{}) - defer teardownDataStore(ds) - - mbName := "fred" - subjects := []string{"a", "br", "much longer than the others"} - sentIds := make([]string, len(subjects)) - sentSizes := make([]int64, len(subjects)) - - for i, subj := range subjects { - // Add a message - id, size := deliverMessage(ds, mbName, subj, time.Now()) - sentIds[i] = id - sentSizes[i] = size - } - - for i, id := range sentIds { - msg, err := ds.GetMessage(mbName, id) - assert.Nil(t, err) - - expect := sentSizes[i] - size := msg.Size() - assert.Equal(t, expect, size, "Expected size of %v, got %v", expect, size) - } - - if t.Failed() { - // Wait for handler to finish logging - time.Sleep(2 * time.Second) - // Dump buffered log data if there was a failure - _, _ = io.Copy(os.Stderr, logbuf) - } -} - // Test missing files func TestFSMissing(t *testing.T) { ds, logbuf := setupDataStore(config.DataStoreConfig{}) diff --git a/pkg/test/storage_suite.go b/pkg/test/storage_suite.go index fd71774..80e8626 100644 --- a/pkg/test/storage_suite.go +++ b/pkg/test/storage_suite.go @@ -1,6 +1,9 @@ package test import ( + "bytes" + "fmt" + "io/ioutil" "net/mail" "strings" "testing" @@ -10,23 +13,37 @@ import ( "github.com/jhillyerd/inbucket/pkg/storage" ) -// StoreSuite runs a set of general tests on the provided Store -func StoreSuite(t *testing.T, store storage.Store) { +// StoreFactory returns a new store for the test suite. +type StoreFactory func() (store storage.Store, destroy func(), err error) + +// StoreSuite runs a set of general tests on the provided Store. +func StoreSuite(t *testing.T, factory StoreFactory) { testCases := []struct { name string test func(*testing.T, storage.Store) }{ {"metadata", testMetadata}, + {"content", testContent}, + {"delivery order", testDeliveryOrder}, + {"size", testSize}, + {"delete", testDelete}, + {"purge", testPurge}, + {"visit mailboxes", testVisitMailboxes}, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + store, destroy, err := factory() + if err != nil { + t.Fatal(err) + } tc.test(t, store) + destroy() }) } } -func testMetadata(t *testing.T, ds storage.Store) { - // Store a message +// testMetadata verifies message metadata is stored and retrieved correctly. +func testMetadata(t *testing.T, store storage.Store) { mailbox := "testmailbox" from := &mail.Address{Name: "From Person", Address: "from@person.com"} to := []*mail.Address{ @@ -38,7 +55,7 @@ func testMetadata(t *testing.T, ds storage.Store) { content := "doesn't matter" delivery := &message.Delivery{ Meta: message.Metadata{ - // ID and Size will be determined by the Store + // ID and Size will be determined by the Store. Mailbox: mailbox, From: from, To: to, @@ -47,15 +64,15 @@ func testMetadata(t *testing.T, ds storage.Store) { }, Reader: strings.NewReader(content), } - id, err := ds.AddMessage(delivery) + id, err := store.AddMessage(delivery) if err != nil { t.Fatal(err) } if id == "" { t.Fatal("Expected AddMessage() to return non-empty ID string") } - // Retrieve and validate the message - sm, err := ds.GetMessage(mailbox, id) + // Retrieve and validate the message. + sm, err := store.GetMessage(mailbox, id) if err != nil { t.Fatal(err) } @@ -87,3 +104,227 @@ func testMetadata(t *testing.T, ds storage.Store) { t.Errorf("got size %v, want: %v", sm.Size(), len(content)) } } + +// testContent generates some binary content and makes sure it is correctly retrieved. +func testContent(t *testing.T, store storage.Store) { + content := make([]byte, 5000) + for i := 0; i < len(content); i++ { + content[i] = byte(i % 256) + } + mailbox := "testmailbox" + from := &mail.Address{Name: "From Person", Address: "from@person.com"} + to := []*mail.Address{ + {Name: "One Person", Address: "one@a.person.com"}, + } + date := time.Now() + subject := "fantastic test subject line" + delivery := &message.Delivery{ + Meta: message.Metadata{ + // ID and Size will be determined by the Store. + Mailbox: mailbox, + From: from, + To: to, + Date: date, + Subject: subject, + }, + Reader: bytes.NewReader(content), + } + id, err := store.AddMessage(delivery) + if err != nil { + t.Fatal(err) + } + // Get and check. + m, err := store.GetMessage(mailbox, id) + if err != nil { + t.Fatal(err) + } + r, err := m.RawReader() + if err != nil { + t.Fatal(err) + } + got, err := ioutil.ReadAll(r) + if err != nil { + t.Fatal(err) + } + if len(got) != len(content) { + t.Errorf("Got len(content) == %v, want: %v", len(got), len(content)) + } + errors := 0 + for i, b := range got { + if b != content[i] { + t.Errorf("Got content[%v] == %v, want: %v", i, b, content[i]) + errors++ + } + if errors > 5 { + t.Fatalf("Too many content errors, aborting test.") + break + } + } +} + +// testDeliveryOrder delivers several messages to the same mailbox, meanwhile querying its contents +// with a new GetMessages call each cycle. +func testDeliveryOrder(t *testing.T, store storage.Store) { + mailbox := "fred" + subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} + for i, subj := range subjects { + // Check mailbox count. + getAndCountMessages(t, store, mailbox, i) + deliverMessage(t, store, mailbox, subj, time.Now()) + } + // Confirm delivery order. + msgs := getAndCountMessages(t, store, mailbox, 5) + for i, want := range subjects { + got := msgs[i].Subject() + if got != want { + t.Errorf("Got subject %q, want %q", got, want) + } + } +} + +// testSize verifies message contnet size metadata values. +func testSize(t *testing.T, store storage.Store) { + mailbox := "fred" + subjects := []string{"a", "br", "much longer than the others"} + sentIds := make([]string, len(subjects)) + sentSizes := make([]int64, len(subjects)) + for i, subj := range subjects { + id, size := deliverMessage(t, store, mailbox, subj, time.Now()) + sentIds[i] = id + sentSizes[i] = size + } + for i, id := range sentIds { + msg, err := store.GetMessage(mailbox, id) + if err != nil { + t.Fatal(err) + } + want := sentSizes[i] + got := msg.Size() + if got != want { + t.Errorf("Got size %v, want: %v", got, want) + } + } +} + +// testDelete creates and deletes some messages. +func testDelete(t *testing.T, store storage.Store) { + mailbox := "fred" + subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} + for _, subj := range subjects { + deliverMessage(t, store, mailbox, subj, time.Now()) + } + msgs := getAndCountMessages(t, store, mailbox, len(subjects)) + // Delete a couple messages. + err := store.RemoveMessage(mailbox, msgs[1].ID()) + if err != nil { + t.Fatal(err) + } + err = store.RemoveMessage(mailbox, msgs[3].ID()) + if err != nil { + t.Fatal(err) + } + // Confirm deletion. + subjects = []string{"alpha", "charlie", "echo"} + msgs = getAndCountMessages(t, store, mailbox, len(subjects)) + for i, want := range subjects { + got := msgs[i].Subject() + if got != want { + t.Errorf("Got subject %q, want %q", got, want) + } + } + // Try appending one more. + deliverMessage(t, store, mailbox, "foxtrot", time.Now()) + subjects = []string{"alpha", "charlie", "echo", "foxtrot"} + msgs = getAndCountMessages(t, store, mailbox, len(subjects)) + for i, want := range subjects { + got := msgs[i].Subject() + if got != want { + t.Errorf("Got subject %q, want %q", got, want) + } + } +} + +// testPurge makes sure mailboxes can be purged. +func testPurge(t *testing.T, store storage.Store) { + mailbox := "fred" + subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"} + for _, subj := range subjects { + deliverMessage(t, store, mailbox, subj, time.Now()) + } + getAndCountMessages(t, store, mailbox, len(subjects)) + // Purge and verify. + err := store.PurgeMessages(mailbox) + if err != nil { + t.Fatal(err) + } + getAndCountMessages(t, store, mailbox, 0) +} + +// testVisitMailboxes creates some mailboxes and confirms the VisitMailboxes method visits all of +// them. +func testVisitMailboxes(t *testing.T, ds storage.Store) { + boxes := []string{"abby", "bill", "christa", "donald", "evelyn"} + for _, name := range boxes { + deliverMessage(t, ds, name, "Old Message", time.Now().Add(-24*time.Hour)) + deliverMessage(t, ds, name, "New Message", time.Now()) + } + seen := 0 + err := ds.VisitMailboxes(func(messages []storage.StoreMessage) bool { + seen++ + count := len(messages) + if count != 2 { + t.Errorf("got: %v messages, want: 2", count) + } + return true + }) + if err != nil { + t.Error(err) + } + if seen != 5 { + t.Errorf("saw %v messages in total, want: 5", seen) + } +} + +// deliverMessage creates and delivers a message to the specific mailbox, returning the size of the +// generated message. +func deliverMessage( + t *testing.T, + store storage.Store, + mailbox string, + subject string, + date time.Time, +) (string, int64) { + t.Helper() + meta := message.Metadata{ + Mailbox: mailbox, + To: []*mail.Address{{Name: "Some Body", Address: "somebody@host"}}, + From: &mail.Address{Name: "Some B. Else", Address: "somebodyelse@host"}, + Subject: subject, + Date: date, + } + testMsg := fmt.Sprintf("To: %s\r\nFrom: %s\r\nSubject: %s\r\n\r\nTest Body\r\n", + meta.To[0].Address, meta.From.Address, subject) + delivery := &message.Delivery{ + Meta: meta, + Reader: ioutil.NopCloser(strings.NewReader(testMsg)), + } + id, err := store.AddMessage(delivery) + if err != nil { + t.Fatal(err) + } + return id, int64(len(testMsg)) +} + +// getAndCountMessages is a test helper that expects to receive count messages or fails the test, it +// also checks return error. +func getAndCountMessages(t *testing.T, s storage.Store, mailbox string, count int) []storage.StoreMessage { + t.Helper() + msgs, err := s.GetMessages(mailbox) + if err != nil { + t.Fatalf("Failed to GetMessages for %q: %v", mailbox, err) + } + if len(msgs) != count { + t.Errorf("Got %v messages, want: %v", len(msgs), count) + } + return msgs +} From d132efd6fadc73a38ae947330cb6acde8b988a72 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 09:18:28 -0700 Subject: [PATCH 15/26] policy: Create new policy package for #84 --- pkg/policy/address.go | 204 +++++++++++++++++++++++++++++ pkg/policy/address_test.go | 138 ++++++++++++++++++++ pkg/rest/apiv1_controller.go | 11 +- pkg/rest/socketv1_controller.go | 4 +- pkg/server/smtp/handler.go | 9 +- pkg/storage/file/fstore.go | 5 +- pkg/stringutil/utils.go | 208 ------------------------------ pkg/stringutil/utils_test.go | 220 +++----------------------------- pkg/webui/mailbox_controller.go | 18 +-- pkg/webui/root_controller.go | 4 +- 10 files changed, 388 insertions(+), 433 deletions(-) create mode 100644 pkg/policy/address.go create mode 100644 pkg/policy/address_test.go diff --git a/pkg/policy/address.go b/pkg/policy/address.go new file mode 100644 index 0000000..7cc85f0 --- /dev/null +++ b/pkg/policy/address.go @@ -0,0 +1,204 @@ +package policy + +import ( + "bytes" + "fmt" + "strings" +) + +// ParseMailboxName takes a localPart string (ex: "user+ext" without "@domain") +// and returns just the mailbox name (ex: "user"). Returns an error if +// localPart contains invalid characters; it won't accept any that must be +// quoted according to RFC3696. +func ParseMailboxName(localPart string) (result string, err error) { + if localPart == "" { + return "", fmt.Errorf("Mailbox name cannot be empty") + } + result = strings.ToLower(localPart) + invalid := make([]byte, 0, 10) + for i := 0; i < len(result); i++ { + c := result[i] + switch { + case 'a' <= c && c <= 'z': + case '0' <= c && c <= '9': + case bytes.IndexByte([]byte("!#$%&'*+-=/?^_`.{|}~"), c) >= 0: + default: + invalid = append(invalid, c) + } + } + if len(invalid) > 0 { + return "", fmt.Errorf("Mailbox name contained invalid character(s): %q", invalid) + } + if idx := strings.Index(result, "+"); idx > -1 { + result = result[0:idx] + } + return result, nil +} + +// ValidateDomainPart returns true if the domain part complies to RFC3696, RFC1035 +func ValidateDomainPart(domain string) bool { + if len(domain) == 0 { + return false + } + if len(domain) > 255 { + return false + } + if domain[len(domain)-1] != '.' { + domain += "." + } + prev := '.' + labelLen := 0 + hasAlphaNum := false + for _, c := range domain { + switch { + case ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || + ('0' <= c && c <= '9') || c == '_': + // Must contain some of these to be a valid label. + hasAlphaNum = true + labelLen++ + case c == '-': + if prev == '.' { + // Cannot lead with hyphen. + return false + } + case c == '.': + if prev == '.' || prev == '-' { + // Cannot end with hyphen or double-dot. + return false + } + if labelLen > 63 { + return false + } + if !hasAlphaNum { + return false + } + labelLen = 0 + hasAlphaNum = false + default: + // Unknown character. + return false + } + prev = c + } + return true +} + +// ParseEmailAddress unescapes an email address, and splits the local part from the domain part. +// An error is returned if the local or domain parts fail validation following the guidelines +// in RFC3696. +func ParseEmailAddress(address string) (local string, domain string, err error) { + if address == "" { + return "", "", fmt.Errorf("Empty address") + } + if len(address) > 320 { + return "", "", fmt.Errorf("Address exceeds 320 characters") + } + if address[0] == '@' { + return "", "", fmt.Errorf("Address cannot start with @ symbol") + } + if address[0] == '.' { + return "", "", fmt.Errorf("Address cannot start with a period") + } + // Loop over address parsing out local part. + buf := new(bytes.Buffer) + prev := byte('.') + inCharQuote := false + inStringQuote := false +LOOP: + for i := 0; i < len(address); i++ { + c := address[i] + switch { + case ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'): + // Letters are OK. + err = buf.WriteByte(c) + if err != nil { + return + } + inCharQuote = false + case '0' <= c && c <= '9': + // Numbers are OK. + err = buf.WriteByte(c) + if err != nil { + return + } + inCharQuote = false + case bytes.IndexByte([]byte("!#$%&'*+-/=?^_`{|}~"), c) >= 0: + // These specials can be used unquoted. + err = buf.WriteByte(c) + if err != nil { + return + } + inCharQuote = false + case c == '.': + // A single period is OK. + if prev == '.' { + // Sequence of periods is not permitted. + return "", "", fmt.Errorf("Sequence of periods is not permitted") + } + err = buf.WriteByte(c) + if err != nil { + return + } + inCharQuote = false + case c == '\\': + inCharQuote = true + case c == '"': + if inCharQuote { + err = buf.WriteByte(c) + if err != nil { + return + } + inCharQuote = false + } else if inStringQuote { + inStringQuote = false + } else { + if i == 0 { + inStringQuote = true + } else { + return "", "", fmt.Errorf("Quoted string can only begin at start of address") + } + } + case c == '@': + if inCharQuote || inStringQuote { + err = buf.WriteByte(c) + if err != nil { + return + } + inCharQuote = false + } else { + // End of local-part. + if i > 128 { + return "", "", fmt.Errorf("Local part must not exceed 128 characters") + } + if prev == '.' { + return "", "", fmt.Errorf("Local part cannot end with a period") + } + domain = address[i+1:] + break LOOP + } + case c > 127: + return "", "", fmt.Errorf("Characters outside of US-ASCII range not permitted") + default: + if inCharQuote || inStringQuote { + err = buf.WriteByte(c) + if err != nil { + return + } + inCharQuote = false + } else { + return "", "", fmt.Errorf("Character %q must be quoted", c) + } + } + prev = c + } + if inCharQuote { + return "", "", fmt.Errorf("Cannot end address with unterminated quoted-pair") + } + if inStringQuote { + return "", "", fmt.Errorf("Cannot end address with unterminated string quote") + } + if !ValidateDomainPart(domain) { + return "", "", fmt.Errorf("Domain part validation failed") + } + return buf.String(), domain, nil +} diff --git a/pkg/policy/address_test.go b/pkg/policy/address_test.go new file mode 100644 index 0000000..149e9b3 --- /dev/null +++ b/pkg/policy/address_test.go @@ -0,0 +1,138 @@ +package policy_test + +import ( + "strings" + "testing" + + "github.com/jhillyerd/inbucket/pkg/policy" +) + +func TestParseMailboxName(t *testing.T) { + var validTable = []struct { + input string + expect string + }{ + {"mailbox", "mailbox"}, + {"user123", "user123"}, + {"MailBOX", "mailbox"}, + {"First.Last", "first.last"}, + {"user+label", "user"}, + {"chars!#$%", "chars!#$%"}, + {"chars&'*-", "chars&'*-"}, + {"chars=/?^", "chars=/?^"}, + {"chars_`.{", "chars_`.{"}, + {"chars|}~", "chars|}~"}, + } + for _, tt := range validTable { + if result, err := policy.ParseMailboxName(tt.input); err != nil { + t.Errorf("Error while parsing %q: %v", tt.input, err) + } else { + if result != tt.expect { + t.Errorf("Parsing %q, expected %q, got %q", tt.input, tt.expect, result) + } + } + } + var invalidTable = []struct { + input, msg string + }{ + {"", "Empty mailbox name is not permitted"}, + {"user@host", "@ symbol not permitted"}, + {"first last", "Space not permitted"}, + {"first\"last", "Double quote not permitted"}, + {"first\nlast", "Control chars not permitted"}, + } + for _, tt := range invalidTable { + if _, err := policy.ParseMailboxName(tt.input); err == nil { + t.Errorf("Didn't get an error while parsing %q: %v", tt.input, tt.msg) + } + } +} + +func TestValidateDomain(t *testing.T) { + var testTable = []struct { + input string + expect bool + msg string + }{ + {"", false, "Empty domain is not valid"}, + {"hostname", true, "Just a hostname is valid"}, + {"github.com", true, "Two labels should be just fine"}, + {"my-domain.com", true, "Hyphen is allowed mid-label"}, + {"_domainkey.foo.com", true, "Underscores are allowed"}, + {"bar.com.", true, "Must be able to end with a dot"}, + {"ABC.6DBS.com", true, "Mixed case is OK"}, + {"mail.123.com", true, "Number only label valid"}, + {"123.com", true, "Number only label valid"}, + {"google..com", false, "Double dot not valid"}, + {".foo.com", false, "Cannot start with a dot"}, + {"google\r.com", false, "Special chars not allowed"}, + {"foo.-bar.com", false, "Label cannot start with hyphen"}, + {"foo-.bar.com", false, "Label cannot end with hyphen"}, + {strings.Repeat("a", 256), false, "Max domain length is 255"}, + {strings.Repeat("a", 63) + ".com", true, "Should allow 63 char domain label"}, + {strings.Repeat("a", 64) + ".com", false, "Max domain label length is 63"}, + } + for _, tt := range testTable { + if policy.ValidateDomainPart(tt.input) != tt.expect { + t.Errorf("Expected %v for %q: %s", tt.expect, tt.input, tt.msg) + } + } +} + +func TestValidateLocal(t *testing.T) { + var testTable = []struct { + input string + expect bool + msg string + }{ + {"", false, "Empty local is not valid"}, + {"a", true, "Single letter should be fine"}, + {strings.Repeat("a", 128), true, "Valid up to 128 characters"}, + {strings.Repeat("a", 129), false, "Only valid up to 128 characters"}, + {"FirstLast", true, "Mixed case permitted"}, + {"user123", true, "Numbers permitted"}, + {"a!#$%&'*+-/=?^_`{|}~", true, "Any of !#$%&'*+-/=?^_`{|}~ are permitted"}, + {"first.last", true, "Embedded period is permitted"}, + {"first..last", false, "Sequence of periods is not allowed"}, + {".user", false, "Cannot lead with a period"}, + {"user.", false, "Cannot end with a period"}, + {"james@mail", false, "Unquoted @ not permitted"}, + {"first last", false, "Unquoted space not permitted"}, + {"tricky\\. ", false, "Unquoted space not permitted"}, + {"no,commas", false, "Unquoted comma not allowed"}, + {"t[es]t", false, "Unquoted square brackets not allowed"}, + {"james\\", false, "Cannot end with backslash quote"}, + {"james\\@mail", true, "Quoted @ permitted"}, + {"quoted\\ space", true, "Quoted space permitted"}, + {"no\\,commas", true, "Quoted comma is OK"}, + {"t\\[es\\]t", true, "Quoted brackets are OK"}, + {"user\\name", true, "Should be able to quote a-z"}, + {"USER\\NAME", true, "Should be able to quote A-Z"}, + {"user\\1", true, "Should be able to quote a digit"}, + {"one\\$\\|", true, "Should be able to quote plain specials"}, + {"return\\\r", true, "Should be able to quote ASCII control chars"}, + {"high\\\x80", false, "Should not accept > 7-bit quoted chars"}, + {"quote\\\"", true, "Quoted double quote is permitted"}, + {"\"james\"", true, "Quoted a-z is permitted"}, + {"\"first last\"", true, "Quoted space is permitted"}, + {"\"quoted@sign\"", true, "Quoted @ is allowed"}, + {"\"qp\\\"quote\"", true, "Quoted quote within quoted string is OK"}, + {"\"unterminated", false, "Quoted string must be terminated"}, + {"\"unterminated\\\"", false, "Quoted string must be terminated"}, + {"embed\"quote\"string", false, "Embedded quoted string is illegal"}, + {"user+mailbox", true, "RFC3696 test case should be valid"}, + {"customer/department=shipping", true, "RFC3696 test case should be valid"}, + {"$A12345", true, "RFC3696 test case should be valid"}, + {"!def!xyz%abc", true, "RFC3696 test case should be valid"}, + {"_somename", true, "RFC3696 test case should be valid"}, + } + for _, tt := range testTable { + _, _, err := policy.ParseEmailAddress(tt.input + "@domain.com") + if (err != nil) == tt.expect { + if err != nil { + t.Logf("Got error: %s", err) + } + t.Errorf("Expected %v for %q: %s", tt.expect, tt.input, tt.msg) + } + } +} diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index 7b40201..2b10eb0 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -11,6 +11,7 @@ import ( "strconv" "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/rest/model" "github.com/jhillyerd/inbucket/pkg/server/web" "github.com/jhillyerd/inbucket/pkg/storage" @@ -20,7 +21,7 @@ import ( // MailboxListV1 renders a list of messages in a mailbox func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -50,7 +51,7 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -100,7 +101,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( // MailboxPurgeV1 deletes all messages from a mailbox func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -118,7 +119,7 @@ func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -142,7 +143,7 @@ func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } diff --git a/pkg/rest/socketv1_controller.go b/pkg/rest/socketv1_controller.go index 5ad2dbf..7614ac1 100644 --- a/pkg/rest/socketv1_controller.go +++ b/pkg/rest/socketv1_controller.go @@ -7,9 +7,9 @@ import ( "github.com/gorilla/websocket" "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/rest/model" "github.com/jhillyerd/inbucket/pkg/server/web" - "github.com/jhillyerd/inbucket/pkg/stringutil" ) const ( @@ -173,7 +173,7 @@ func MonitorAllMessagesV1( // notifies the client of messages received by a particular mailbox. func MonitorMailboxMessagesV1( w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index d9dad39..e123e9f 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -16,6 +16,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/stringutil" ) @@ -267,7 +268,7 @@ func (ss *Session) readyHandler(cmd string, arg string) { return } from := m[1] - if _, _, err := stringutil.ParseEmailAddress(from); err != nil { + if _, _, err := policy.ParseEmailAddress(from); err != nil { ss.send("501 Bad sender address syntax") ss.logWarn("Bad address as MAIL arg: %q, %s", from, err) return @@ -316,7 +317,7 @@ func (ss *Session) mailHandler(cmd string, arg string) { } // This trim is probably too forgiving recip := strings.Trim(arg[3:], "<> ") - if _, _, err := stringutil.ParseEmailAddress(recip); err != nil { + if _, _, err := policy.ParseEmailAddress(recip); err != nil { ss.send("501 Bad recipient address syntax") ss.logWarn("Bad address as RCPT arg: %q, %s", recip, err) return @@ -355,7 +356,7 @@ func (ss *Session) dataHandler() { if ss.server.storeMessages { for e := ss.recipients.Front(); e != nil; e = e.Next() { recip := e.Value.(string) - local, domain, err := stringutil.ParseEmailAddress(recip) + local, domain, err := policy.ParseEmailAddress(recip) if err != nil { ss.logError("Failed to parse address for %q", recip) ss.send(fmt.Sprintf("451 Failed to open mailbox for %v", recip)) @@ -436,7 +437,7 @@ func (ss *Session) dataHandler() { // deliverMessage creates and populates a new Message for the specified recipient func (ss *Session) deliverMessage(r recipientDetails, content []byte) (ok bool) { - name, err := stringutil.ParseMailboxName(r.localPart) + name, err := policy.ParseMailboxName(r.localPart) if err != nil { // This parse already succeeded when MailboxFor was called, shouldn't fail here. return false diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index b41b5d3..dc20b5d 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -13,6 +13,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/stringutil" ) @@ -218,7 +219,7 @@ func (fs *Store) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) erro // LockFor returns the RWMutex for this mailbox, or an error. func (fs *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { - name, err := stringutil.ParseMailboxName(emailAddress) + name, err := policy.ParseMailboxName(emailAddress) if err != nil { return nil, err } @@ -237,7 +238,7 @@ func (fs *Store) NewMessage(mailbox string) (storage.StoreMessage, error) { // mbox returns the named mailbox. func (fs *Store) mbox(mailbox string) (*mbox, error) { - name, err := stringutil.ParseMailboxName(mailbox) + name, err := policy.ParseMailboxName(mailbox) if err != nil { return nil, err } diff --git a/pkg/stringutil/utils.go b/pkg/stringutil/utils.go index 193e552..699eff8 100644 --- a/pkg/stringutil/utils.go +++ b/pkg/stringutil/utils.go @@ -1,47 +1,12 @@ package stringutil import ( - "bytes" "crypto/sha1" "fmt" "io" "net/mail" - "strings" ) -// ParseMailboxName takes a localPart string (ex: "user+ext" without "@domain") -// and returns just the mailbox name (ex: "user"). Returns an error if -// localPart contains invalid characters; it won't accept any that must be -// quoted according to RFC3696. -func ParseMailboxName(localPart string) (result string, err error) { - if localPart == "" { - return "", fmt.Errorf("Mailbox name cannot be empty") - } - result = strings.ToLower(localPart) - - invalid := make([]byte, 0, 10) - - for i := 0; i < len(result); i++ { - c := result[i] - switch { - case 'a' <= c && c <= 'z': - case '0' <= c && c <= '9': - case bytes.IndexByte([]byte("!#$%&'*+-=/?^_`.{|}~"), c) >= 0: - default: - invalid = append(invalid, c) - } - } - - if len(invalid) > 0 { - return "", fmt.Errorf("Mailbox name contained invalid character(s): %q", invalid) - } - - if idx := strings.Index(result, "+"); idx > -1 { - result = result[0:idx] - } - return result, nil -} - // HashMailboxName accepts a mailbox name and hashes it. filestore uses this as // the directory to house the mailbox func HashMailboxName(mailbox string) string { @@ -53,179 +18,6 @@ func HashMailboxName(mailbox string) string { return fmt.Sprintf("%x", h.Sum(nil)) } -// ValidateDomainPart returns true if the domain part complies to RFC3696, RFC1035 -func ValidateDomainPart(domain string) bool { - if len(domain) == 0 { - return false - } - if len(domain) > 255 { - return false - } - if domain[len(domain)-1] != '.' { - domain += "." - } - prev := '.' - labelLen := 0 - hasAlphaNum := false - - for _, c := range domain { - switch { - case ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || - ('0' <= c && c <= '9') || c == '_': - // Must contain some of these to be a valid label - hasAlphaNum = true - labelLen++ - case c == '-': - if prev == '.' { - // Cannot lead with hyphen - return false - } - case c == '.': - if prev == '.' || prev == '-' { - // Cannot end with hyphen or double-dot - return false - } - if labelLen > 63 { - return false - } - if !hasAlphaNum { - return false - } - labelLen = 0 - hasAlphaNum = false - default: - // Unknown character - return false - } - prev = c - } - - return true -} - -// ParseEmailAddress unescapes an email address, and splits the local part from the domain part. -// An error is returned if the local or domain parts fail validation following the guidelines -// in RFC3696. -func ParseEmailAddress(address string) (local string, domain string, err error) { - if address == "" { - return "", "", fmt.Errorf("Empty address") - } - if len(address) > 320 { - return "", "", fmt.Errorf("Address exceeds 320 characters") - } - if address[0] == '@' { - return "", "", fmt.Errorf("Address cannot start with @ symbol") - } - if address[0] == '.' { - return "", "", fmt.Errorf("Address cannot start with a period") - } - - // Loop over address parsing out local part - buf := new(bytes.Buffer) - prev := byte('.') - inCharQuote := false - inStringQuote := false -LOOP: - for i := 0; i < len(address); i++ { - c := address[i] - switch { - case ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'): - // Letters are OK - err = buf.WriteByte(c) - if err != nil { - return - } - inCharQuote = false - case '0' <= c && c <= '9': - // Numbers are OK - err = buf.WriteByte(c) - if err != nil { - return - } - inCharQuote = false - case bytes.IndexByte([]byte("!#$%&'*+-/=?^_`{|}~"), c) >= 0: - // These specials can be used unquoted - err = buf.WriteByte(c) - if err != nil { - return - } - inCharQuote = false - case c == '.': - // A single period is OK - if prev == '.' { - // Sequence of periods is not permitted - return "", "", fmt.Errorf("Sequence of periods is not permitted") - } - err = buf.WriteByte(c) - if err != nil { - return - } - inCharQuote = false - case c == '\\': - inCharQuote = true - case c == '"': - if inCharQuote { - err = buf.WriteByte(c) - if err != nil { - return - } - inCharQuote = false - } else if inStringQuote { - inStringQuote = false - } else { - if i == 0 { - inStringQuote = true - } else { - return "", "", fmt.Errorf("Quoted string can only begin at start of address") - } - } - case c == '@': - if inCharQuote || inStringQuote { - err = buf.WriteByte(c) - if err != nil { - return - } - inCharQuote = false - } else { - // End of local-part - if i > 128 { - return "", "", fmt.Errorf("Local part must not exceed 128 characters") - } - if prev == '.' { - return "", "", fmt.Errorf("Local part cannot end with a period") - } - domain = address[i+1:] - break LOOP - } - case c > 127: - return "", "", fmt.Errorf("Characters outside of US-ASCII range not permitted") - default: - if inCharQuote || inStringQuote { - err = buf.WriteByte(c) - if err != nil { - return - } - inCharQuote = false - } else { - return "", "", fmt.Errorf("Character %q must be quoted", c) - } - } - prev = c - } - if inCharQuote { - return "", "", fmt.Errorf("Cannot end address with unterminated quoted-pair") - } - if inStringQuote { - return "", "", fmt.Errorf("Cannot end address with unterminated string quote") - } - - if !ValidateDomainPart(domain) { - return "", "", fmt.Errorf("Domain part validation failed") - } - - return buf.String(), domain, nil -} - // StringAddressList converts a list of addresses to a list of strings func StringAddressList(addrs []*mail.Address) []string { s := make([]string, len(addrs)) diff --git a/pkg/stringutil/utils_test.go b/pkg/stringutil/utils_test.go index 330bfbd..8c0b7bd 100644 --- a/pkg/stringutil/utils_test.go +++ b/pkg/stringutil/utils_test.go @@ -1,215 +1,33 @@ -package stringutil +package stringutil_test import ( - "strings" + "net/mail" "testing" - "github.com/stretchr/testify/assert" + "github.com/jhillyerd/inbucket/pkg/stringutil" ) -func TestParseMailboxName(t *testing.T) { - var validTable = []struct { - input string - expect string - }{ - {"mailbox", "mailbox"}, - {"user123", "user123"}, - {"MailBOX", "mailbox"}, - {"First.Last", "first.last"}, - {"user+label", "user"}, - {"chars!#$%", "chars!#$%"}, - {"chars&'*-", "chars&'*-"}, - {"chars=/?^", "chars=/?^"}, - {"chars_`.{", "chars_`.{"}, - {"chars|}~", "chars|}~"}, - } - - for _, tt := range validTable { - if result, err := ParseMailboxName(tt.input); err != nil { - t.Errorf("Error while parsing %q: %v", tt.input, err) - } else { - if result != tt.expect { - t.Errorf("Parsing %q, expected %q, got %q", tt.input, tt.expect, result) - } - } - } - - var invalidTable = []struct { - input, msg string - }{ - {"", "Empty mailbox name is not permitted"}, - {"user@host", "@ symbol not permitted"}, - {"first last", "Space not permitted"}, - {"first\"last", "Double quote not permitted"}, - {"first\nlast", "Control chars not permitted"}, - } - - for _, tt := range invalidTable { - if _, err := ParseMailboxName(tt.input); err == nil { - t.Errorf("Didn't get an error while parsing %q: %v", tt.input, tt.msg) - } - } -} - func TestHashMailboxName(t *testing.T) { - assert.Equal(t, HashMailboxName("mail"), "1d6e1cf70ec6f9ab28d3ea4b27a49a77654d370e") -} - -func TestValidateDomain(t *testing.T) { - assert.False(t, ValidateDomainPart(strings.Repeat("a", 256)), - "Max domain length is 255") - assert.False(t, ValidateDomainPart(strings.Repeat("a", 64)+".com"), - "Max label length is 63") - assert.True(t, ValidateDomainPart(strings.Repeat("a", 63)+".com"), - "Should allow 63 char label") - - var testTable = []struct { - input string - expect bool - msg string - }{ - {"", false, "Empty domain is not valid"}, - {"hostname", true, "Just a hostname is valid"}, - {"github.com", true, "Two labels should be just fine"}, - {"my-domain.com", true, "Hyphen is allowed mid-label"}, - {"_domainkey.foo.com", true, "Underscores are allowed"}, - {"bar.com.", true, "Must be able to end with a dot"}, - {"ABC.6DBS.com", true, "Mixed case is OK"}, - {"mail.123.com", true, "Number only label valid"}, - {"123.com", true, "Number only label valid"}, - {"google..com", false, "Double dot not valid"}, - {".foo.com", false, "Cannot start with a dot"}, - {"google\r.com", false, "Special chars not allowed"}, - {"foo.-bar.com", false, "Label cannot start with hyphen"}, - {"foo-.bar.com", false, "Label cannot end with hyphen"}, - } - - for _, tt := range testTable { - if ValidateDomainPart(tt.input) != tt.expect { - t.Errorf("Expected %v for %q: %s", tt.expect, tt.input, tt.msg) - } + want := "1d6e1cf70ec6f9ab28d3ea4b27a49a77654d370e" + got := stringutil.HashMailboxName("mail") + if got != want { + t.Errorf("Got %q, want %q", got, want) } } -func TestValidateLocal(t *testing.T) { - var testTable = []struct { - input string - expect bool - msg string - }{ - {"", false, "Empty local is not valid"}, - {"a", true, "Single letter should be fine"}, - {strings.Repeat("a", 128), true, "Valid up to 128 characters"}, - {strings.Repeat("a", 129), false, "Only valid up to 128 characters"}, - {"FirstLast", true, "Mixed case permitted"}, - {"user123", true, "Numbers permitted"}, - {"a!#$%&'*+-/=?^_`{|}~", true, "Any of !#$%&'*+-/=?^_`{|}~ are permitted"}, - {"first.last", true, "Embedded period is permitted"}, - {"first..last", false, "Sequence of periods is not allowed"}, - {".user", false, "Cannot lead with a period"}, - {"user.", false, "Cannot end with a period"}, - {"james@mail", false, "Unquoted @ not permitted"}, - {"first last", false, "Unquoted space not permitted"}, - {"tricky\\. ", false, "Unquoted space not permitted"}, - {"no,commas", false, "Unquoted comma not allowed"}, - {"t[es]t", false, "Unquoted square brackets not allowed"}, - {"james\\", false, "Cannot end with backslash quote"}, - {"james\\@mail", true, "Quoted @ permitted"}, - {"quoted\\ space", true, "Quoted space permitted"}, - {"no\\,commas", true, "Quoted comma is OK"}, - {"t\\[es\\]t", true, "Quoted brackets are OK"}, - {"user\\name", true, "Should be able to quote a-z"}, - {"USER\\NAME", true, "Should be able to quote A-Z"}, - {"user\\1", true, "Should be able to quote a digit"}, - {"one\\$\\|", true, "Should be able to quote plain specials"}, - {"return\\\r", true, "Should be able to quote ASCII control chars"}, - {"high\\\x80", false, "Should not accept > 7-bit quoted chars"}, - {"quote\\\"", true, "Quoted double quote is permitted"}, - {"\"james\"", true, "Quoted a-z is permitted"}, - {"\"first last\"", true, "Quoted space is permitted"}, - {"\"quoted@sign\"", true, "Quoted @ is allowed"}, - {"\"qp\\\"quote\"", true, "Quoted quote within quoted string is OK"}, - {"\"unterminated", false, "Quoted string must be terminated"}, - {"\"unterminated\\\"", false, "Quoted string must be terminated"}, - {"embed\"quote\"string", false, "Embedded quoted string is illegal"}, - - {"user+mailbox", true, "RFC3696 test case should be valid"}, - {"customer/department=shipping", true, "RFC3696 test case should be valid"}, - {"$A12345", true, "RFC3696 test case should be valid"}, - {"!def!xyz%abc", true, "RFC3696 test case should be valid"}, - {"_somename", true, "RFC3696 test case should be valid"}, +func TestStringAddressList(t *testing.T) { + input := []*mail.Address{ + {Name: "Fred B. Fish", Address: "fred@fish.org"}, + {Name: "User", Address: "user@domain.org"}, } - - for _, tt := range testTable { - _, _, err := ParseEmailAddress(tt.input + "@domain.com") - if (err != nil) == tt.expect { - if err != nil { - t.Logf("Got error: %s", err) - } - t.Errorf("Expected %v for %q: %s", tt.expect, tt.input, tt.msg) - } - } -} - -func TestParseEmailAddress(t *testing.T) { - // Test some good email addresses - var testTable = []struct { - input, local, domain string - }{ - {"root@localhost", "root", "localhost"}, - {"FirstLast@domain.local", "FirstLast", "domain.local"}, - {"route66@prodigy.net", "route66", "prodigy.net"}, - {"lorbit!user@uucp", "lorbit!user", "uucp"}, - {"user+spam@gmail.com", "user+spam", "gmail.com"}, - {"first.last@domain.local", "first.last", "domain.local"}, - {"first\\ last@_key.domain.com", "first last", "_key.domain.com"}, - {"first\\\"last@a.b.c", "first\"last", "a.b.c"}, - {"user\\@internal@myhost.ca", "user@internal", "myhost.ca"}, - {"\"first last@evil\"@top-secret.gov", "first last@evil", "top-secret.gov"}, - {"\"line\nfeed\"@linenoise.co.uk", "line\nfeed", "linenoise.co.uk"}, - {"user+mailbox@host", "user+mailbox", "host"}, - {"customer/department=shipping@host", "customer/department=shipping", "host"}, - {"$A12345@host", "$A12345", "host"}, - {"!def!xyz%abc@host", "!def!xyz%abc", "host"}, - {"_somename@host", "_somename", "host"}, - } - - for _, tt := range testTable { - local, domain, err := ParseEmailAddress(tt.input) - if err != nil { - t.Errorf("Error when parsing %q: %s", tt.input, err) - } else { - if tt.local != local { - t.Errorf("When parsing %q, expected local %q, got %q instead", - tt.input, tt.local, local) - } - if tt.domain != domain { - t.Errorf("When parsing %q, expected domain %q, got %q instead", - tt.input, tt.domain, domain) - } - } - } - - // Check that validations fail correctly - var badTable = []struct { - input, msg string - }{ - {"", "Empty address not permitted"}, - {"user", "Missing domain part"}, - {"@host", "Missing local part"}, - {"user\\@host", "Missing domain part"}, - {"\"user@host\"", "Missing domain part"}, - {"\"user@host", "Unterminated quoted string"}, - {"first last@host", "Unquoted space"}, - {"user@bad!domain", "Invalid domain"}, - {".user@host", "Can't lead with a ."}, - {"user.@host", "Can't end local with a dot"}, - {"user@bad domain", "No spaces in domain permitted"}, - } - - for _, tt := range badTable { - if _, _, err := ParseEmailAddress(tt.input); err == nil { - t.Errorf("Did not get expected error when parsing %q: %s", tt.input, tt.msg) + want := []string{`"Fred B. Fish" `, `"User" `} + output := stringutil.StringAddressList(input) + if len(output) != len(want) { + t.Fatalf("Got %v strings, want: %v", len(output), len(want)) + } + for i, got := range output { + if got != want[i] { + t.Errorf("Got %q, want: %q", got, want[i]) } } } diff --git a/pkg/webui/mailbox_controller.go b/pkg/webui/mailbox_controller.go index 16aa305..2af9ef1 100644 --- a/pkg/webui/mailbox_controller.go +++ b/pkg/webui/mailbox_controller.go @@ -8,9 +8,9 @@ import ( "strconv" "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/server/web" "github.com/jhillyerd/inbucket/pkg/storage" - "github.com/jhillyerd/inbucket/pkg/stringutil" "github.com/jhillyerd/inbucket/pkg/webui/sanitize" ) @@ -25,7 +25,7 @@ func MailboxIndex(w http.ResponseWriter, req *http.Request, ctx *web.Context) (e http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - name, err = stringutil.ParseMailboxName(name) + name, err = policy.ParseMailboxName(name) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) @@ -52,7 +52,7 @@ func MailboxIndex(w http.ResponseWriter, req *http.Request, ctx *web.Context) (e func MailboxLink(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) @@ -68,7 +68,7 @@ func MailboxLink(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er // MailboxList renders a list of messages in a mailbox. Renders a partial func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -90,7 +90,7 @@ func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -131,7 +131,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -159,7 +159,7 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { return err } @@ -183,7 +183,7 @@ func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) @@ -225,7 +225,7 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Co // MailboxViewAttach sends the attachment to the client for online viewing func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) diff --git a/pkg/webui/root_controller.go b/pkg/webui/root_controller.go index 85d1662..21d4dd3 100644 --- a/pkg/webui/root_controller.go +++ b/pkg/webui/root_controller.go @@ -7,8 +7,8 @@ import ( "net/http" "github.com/jhillyerd/inbucket/pkg/config" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/server/web" - "github.com/jhillyerd/inbucket/pkg/stringutil" ) // RootIndex serves the Inbucket landing page @@ -58,7 +58,7 @@ func RootMonitorMailbox(w http.ResponseWriter, req *http.Request, ctx *web.Conte http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - name, err := stringutil.ParseMailboxName(ctx.Vars["name"]) + name, err := policy.ParseMailboxName(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) From 469a778d81923cc221cc6dc3fd4af65fe97ba978 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 11:15:17 -0700 Subject: [PATCH 16/26] policy: Impl Addressing{} and Recipient{} for #84 --- Makefile | 2 +- pkg/policy/address.go | 136 ++++++++++++++++++++++++------------- pkg/policy/address_test.go | 53 +++++++++++++++ pkg/policy/recipient.go | 25 +++++++ 4 files changed, 167 insertions(+), 49 deletions(-) create mode 100644 pkg/policy/recipient.go diff --git a/Makefile b/Makefile index eacf4ba..ff275c0 100644 --- a/Makefile +++ b/Makefile @@ -36,4 +36,4 @@ lint: @go vet $(PKGS) reflex: - reflex -r '\.go$$' -- sh -c 'echo; date; echo; go test ./...' + reflex -r '\.go$$' -- sh -c 'echo; date; echo; go test ./... && echo ALL PASS' diff --git a/pkg/policy/address.go b/pkg/policy/address.go index 7cc85f0..34ed8f8 100644 --- a/pkg/policy/address.go +++ b/pkg/policy/address.go @@ -3,9 +3,48 @@ package policy import ( "bytes" "fmt" + "net/mail" "strings" + + "github.com/jhillyerd/inbucket/pkg/config" ) +// Addressing handles email address policy. +type Addressing struct { + Config config.SMTPConfig +} + +// NewRecipient parses an address into a Recipient. +func (a *Addressing) NewRecipient(address string) (*Recipient, error) { + local, domain, err := ParseEmailAddress(address) + if err != nil { + return nil, err + } + mailbox, err := ParseMailboxName(local) + if err != nil { + return nil, err + } + ar, err := mail.ParseAddress(address) + if err != nil { + return nil, err + } + return &Recipient{ + Address: *ar, + apolicy: a, + LocalPart: local, + Domain: domain, + Mailbox: mailbox, + }, nil +} + +// ShouldStoreDomain indicates if Inbucket stores email destined for the specified domain. +func (a *Addressing) ShouldStoreDomain(domain string) bool { + if a.Config.StoreMessages { + return strings.ToLower(domain) != strings.ToLower(a.Config.DomainNoStore) + } + return false +} + // ParseMailboxName takes a localPart string (ex: "user+ext" without "@domain") // and returns just the mailbox name (ex: "user"). Returns an error if // localPart contains invalid characters; it won't accept any that must be @@ -35,54 +74,6 @@ func ParseMailboxName(localPart string) (result string, err error) { return result, nil } -// ValidateDomainPart returns true if the domain part complies to RFC3696, RFC1035 -func ValidateDomainPart(domain string) bool { - if len(domain) == 0 { - return false - } - if len(domain) > 255 { - return false - } - if domain[len(domain)-1] != '.' { - domain += "." - } - prev := '.' - labelLen := 0 - hasAlphaNum := false - for _, c := range domain { - switch { - case ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || - ('0' <= c && c <= '9') || c == '_': - // Must contain some of these to be a valid label. - hasAlphaNum = true - labelLen++ - case c == '-': - if prev == '.' { - // Cannot lead with hyphen. - return false - } - case c == '.': - if prev == '.' || prev == '-' { - // Cannot end with hyphen or double-dot. - return false - } - if labelLen > 63 { - return false - } - if !hasAlphaNum { - return false - } - labelLen = 0 - hasAlphaNum = false - default: - // Unknown character. - return false - } - prev = c - } - return true -} - // ParseEmailAddress unescapes an email address, and splits the local part from the domain part. // An error is returned if the local or domain parts fail validation following the guidelines // in RFC3696. @@ -202,3 +193,52 @@ LOOP: } return buf.String(), domain, nil } + +// ValidateDomainPart returns true if the domain part complies to RFC3696, RFC1035. Used by +// ParseEmailAddress(). +func ValidateDomainPart(domain string) bool { + if len(domain) == 0 { + return false + } + if len(domain) > 255 { + return false + } + if domain[len(domain)-1] != '.' { + domain += "." + } + prev := '.' + labelLen := 0 + hasAlphaNum := false + for _, c := range domain { + switch { + case ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || + ('0' <= c && c <= '9') || c == '_': + // Must contain some of these to be a valid label. + hasAlphaNum = true + labelLen++ + case c == '-': + if prev == '.' { + // Cannot lead with hyphen. + return false + } + case c == '.': + if prev == '.' || prev == '-' { + // Cannot end with hyphen or double-dot. + return false + } + if labelLen > 63 { + return false + } + if !hasAlphaNum { + return false + } + labelLen = 0 + hasAlphaNum = false + default: + // Unknown character. + return false + } + prev = c + } + return true +} diff --git a/pkg/policy/address_test.go b/pkg/policy/address_test.go index 149e9b3..2ab2c60 100644 --- a/pkg/policy/address_test.go +++ b/pkg/policy/address_test.go @@ -4,9 +4,62 @@ import ( "strings" "testing" + "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/policy" ) +func TestShouldStoreDomain(t *testing.T) { + // Test with storage enabled. + ap := &policy.Addressing{ + Config: config.SMTPConfig{ + DomainNoStore: "Foo.Com", + StoreMessages: true, + }, + } + testCases := []struct { + domain string + want bool + }{ + {domain: "bar.com", want: true}, + {domain: "foo.com", want: false}, + {domain: "FOO.com", want: false}, + {domain: "bar.foo.com", want: true}, + } + for _, tc := range testCases { + t.Run(tc.domain, func(t *testing.T) { + got := ap.ShouldStoreDomain(tc.domain) + if got != tc.want { + t.Errorf("Got %v for %q, want: %v", got, tc.domain, tc.want) + } + + }) + } + // Test with storage disabled. + ap = &policy.Addressing{ + Config: config.SMTPConfig{ + StoreMessages: false, + }, + } + testCases = []struct { + domain string + want bool + }{ + {domain: "bar.com", want: false}, + {domain: "foo.com", want: false}, + {domain: "FOO.com", want: false}, + {domain: "bar.foo.com", want: false}, + } + for _, tc := range testCases { + t.Run(tc.domain, func(t *testing.T) { + got := ap.ShouldStoreDomain(tc.domain) + if got != tc.want { + t.Errorf("Got %v for %q, want: %v", got, tc.domain, tc.want) + } + + }) + } +} + func TestParseMailboxName(t *testing.T) { var validTable = []struct { input string diff --git a/pkg/policy/recipient.go b/pkg/policy/recipient.go new file mode 100644 index 0000000..36fd94b --- /dev/null +++ b/pkg/policy/recipient.go @@ -0,0 +1,25 @@ +package policy + +import "net/mail" + +// Recipient represents a potential email recipient, allows policies for it to be queried. +type Recipient struct { + mail.Address + apolicy *Addressing + // LocalPart is the part of the address before @, including +extension. + LocalPart string + // Domain is the part of the address after @. + Domain string + // Mailbox is the canonical mailbox name for this recipient. + Mailbox string +} + +// ShouldAccept returns true if Inbucket should accept mail for this recipient. +func (r *Recipient) ShouldAccept() bool { + return true +} + +// ShouldStore returns true if Inbucket should store mail for this recipient. +func (r *Recipient) ShouldStore() bool { + return r.apolicy.ShouldStoreDomain(r.Domain) +} From b9003a9328ac7a939b2e044c55b9101d550f0dfb Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 12:39:09 -0700 Subject: [PATCH 17/26] smtp: Wire in policy.Recipient for #84 --- cmd/inbucket/main.go | 4 +- pkg/server/smtp/handler.go | 90 ++++++++++++--------------------- pkg/server/smtp/handler_test.go | 9 ++-- pkg/server/smtp/listener.go | 10 ++-- 4 files changed, 47 insertions(+), 66 deletions(-) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index 7020676..ff5a313 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -16,6 +16,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/rest" "github.com/jhillyerd/inbucket/pkg/server/pop3" "github.com/jhillyerd/inbucket/pkg/server/smtp" @@ -135,7 +136,8 @@ func main() { go pop3Server.Start(rootCtx) // Startup SMTP server - smtpServer = smtp.NewServer(config.GetSMTPConfig(), shutdownChan, ds, msgHub) + apolicy := &policy.Addressing{Config: config.GetSMTPConfig()} + smtpServer = smtp.NewServer(config.GetSMTPConfig(), shutdownChan, ds, apolicy, msgHub) go smtpServer.Start(rootCtx) // Loop forever waiting for signals or shutdown channel diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index e123e9f..f15656b 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -3,7 +3,6 @@ package smtp import ( "bufio" "bytes" - "container/list" "fmt" "io" "net" @@ -72,11 +71,6 @@ var commands = map[string]bool{ "TURN": true, } -// recipientDetails for message delivery -type recipientDetails struct { - address, localPart, domainPart string -} - // Session holds the state of an SMTP session type Session struct { server *Server @@ -88,14 +82,22 @@ type Session struct { state State reader *bufio.Reader from string - recipients *list.List + recipients []*policy.Recipient } // NewSession creates a new Session for the given connection func NewSession(server *Server, id int, conn net.Conn) *Session { reader := bufio.NewReader(conn) host, _, _ := net.SplitHostPort(conn.RemoteAddr().String()) - return &Session{server: server, id: id, conn: conn, state: GREET, reader: reader, remoteHost: host} + return &Session{ + server: server, + id: id, + conn: conn, + state: GREET, + reader: reader, + remoteHost: host, + recipients: make([]*policy.Recipient, 0), + } } func (ss *Session) String() string { @@ -297,7 +299,6 @@ func (ss *Session) readyHandler(cmd string, arg string) { } } ss.from = from - ss.recipients = list.New() ss.logInfo("Mail from: %v", from) ss.send(fmt.Sprintf("250 Roger, accepting mail from <%v>", from)) ss.enterState(MAIL) @@ -316,20 +317,21 @@ func (ss *Session) mailHandler(cmd string, arg string) { return } // This trim is probably too forgiving - recip := strings.Trim(arg[3:], "<> ") - if _, _, err := policy.ParseEmailAddress(recip); err != nil { + addr := strings.Trim(arg[3:], "<> ") + recip, err := ss.server.apolicy.NewRecipient(addr) + if err != nil { ss.send("501 Bad recipient address syntax") - ss.logWarn("Bad address as RCPT arg: %q, %s", recip, err) + ss.logWarn("Bad address as RCPT arg: %q, %s", addr, err) return } - if ss.recipients.Len() >= ss.server.maxRecips { + if len(ss.recipients) >= ss.server.maxRecips { ss.logWarn("Maximum limit of %v recipients reached", ss.server.maxRecips) ss.send(fmt.Sprintf("552 Maximum limit of %v recipients reached", ss.server.maxRecips)) return } - ss.recipients.PushBack(recip) - ss.logInfo("Recipient: %v", recip) - ss.send(fmt.Sprintf("250 I'll make sure <%v> gets this", recip)) + ss.recipients = append(ss.recipients, recip) + ss.logInfo("Recipient: %v", addr) + ss.send(fmt.Sprintf("250 I'll make sure <%v> gets this", addr)) return case "DATA": if arg != "" { @@ -337,7 +339,7 @@ func (ss *Session) mailHandler(cmd string, arg string) { ss.logWarn("Got unexpected args on DATA: %q", arg) return } - if ss.recipients.Len() > 0 { + if len(ss.recipients) > 0 { // We have recipients, go to accept data ss.enterState(DATA) return @@ -351,27 +353,6 @@ func (ss *Session) mailHandler(cmd string, arg string) { // DATA func (ss *Session) dataHandler() { - recipients := make([]recipientDetails, 0, ss.recipients.Len()) - // Get a Mailbox and a new Message for each recipient - if ss.server.storeMessages { - for e := ss.recipients.Front(); e != nil; e = e.Next() { - recip := e.Value.(string) - local, domain, err := policy.ParseEmailAddress(recip) - if err != nil { - ss.logError("Failed to parse address for %q", recip) - ss.send(fmt.Sprintf("451 Failed to open mailbox for %v", recip)) - ss.reset() - return - } - if strings.ToLower(domain) != ss.server.domainNoStore { - // Not our "no store" domain, so store the message - recipients = append(recipients, recipientDetails{recip, local, domain}) - } else { - log.Tracef("Not storing message for %q", recip) - } - } - } - ss.send("354 Start mail input; end with .") msgBuf := &bytes.Buffer{} for { @@ -388,31 +369,27 @@ func (ss *Session) dataHandler() { } if bytes.Equal(lineBuf, []byte(".\r\n")) || bytes.Equal(lineBuf, []byte(".\n")) { // Mail data complete - if ss.server.storeMessages { - // Create a message for each valid recipient - for _, r := range recipients { + for _, recip := range ss.recipients { + if recip.ShouldStore() { // TODO temporary hack to fix #77 until datastore revamp - mu, err := ss.server.dataStore.LockFor(r.localPart) + mu, err := ss.server.dataStore.LockFor(recip.LocalPart) if err != nil { - ss.logError("Failed to get lock for %q: %s", r.localPart, err) + ss.logError("Failed to get lock for %q: %s", recip.LocalPart, err) // Delivery failure - ss.send(fmt.Sprintf("451 Failed to store message for %v", r.localPart)) + ss.send(fmt.Sprintf("451 Failed to store message for %v", recip.LocalPart)) ss.reset() return } mu.Lock() - ok := ss.deliverMessage(r, msgBuf.Bytes()) + ok := ss.deliverMessage(recip, msgBuf.Bytes()) mu.Unlock() - if ok { - expReceivedTotal.Add(1) - } else { + if !ok { // Delivery failure - ss.send(fmt.Sprintf("451 Failed to store message for %v", r.localPart)) + ss.send(fmt.Sprintf("451 Failed to store message for %v", recip.LocalPart)) ss.reset() return } } - } else { expReceivedTotal.Add(1) } ss.send("250 Mail accepted for delivery") @@ -436,8 +413,8 @@ func (ss *Session) dataHandler() { } // deliverMessage creates and populates a new Message for the specified recipient -func (ss *Session) deliverMessage(r recipientDetails, content []byte) (ok bool) { - name, err := policy.ParseMailboxName(r.localPart) +func (ss *Session) deliverMessage(recip *policy.Recipient, content []byte) (ok bool) { + name, err := policy.ParseMailboxName(recip.LocalPart) if err != nil { // This parse already succeeded when MailboxFor was called, shouldn't fail here. return false @@ -445,7 +422,7 @@ func (ss *Session) deliverMessage(r recipientDetails, content []byte) (ok bool) // TODO replace with something that only reads header? env, err := enmime.ReadEnvelope(bytes.NewReader(content)) if err != nil { - ss.logError("Failed to parse message for %q: %v", r.localPart, err) + ss.logError("Failed to parse message for %q: %v", recip.LocalPart, err) return false } from, err := env.AddressList("From") @@ -461,7 +438,7 @@ func (ss *Session) deliverMessage(r recipientDetails, content []byte) (ok bool) // Generate Received header. stamp := time.Now().Format(timeStampFormat) recd := strings.NewReader(fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", - ss.remoteDomain, ss.remoteHost, ss.server.domain, r.address, stamp)) + ss.remoteDomain, ss.remoteHost, ss.server.domain, recip.Address, stamp)) delivery := &message.Delivery{ Meta: message.Metadata{ Mailbox: name, @@ -474,7 +451,7 @@ func (ss *Session) deliverMessage(r recipientDetails, content []byte) (ok bool) } id, err := ss.server.dataStore.AddMessage(delivery) if err != nil { - ss.logError("Failed to store message for %q: %s", r.localPart, err) + ss.logError("Failed to store message for %q: %s", recip.LocalPart, err) return false } // Broadcast message information. @@ -519,8 +496,7 @@ func (ss *Session) send(msg string) { ss.logTrace(">> %v >>", msg) } -// readByteLine reads a line of input into the provided buffer. Does -// not reset the Buffer - please do so prior to calling. +// readByteLine reads a line of input, returns byte slice. func (ss *Session) readByteLine() ([]byte, error) { if err := ss.conn.SetReadDeadline(ss.nextDeadline()); err != nil { return nil, err diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index 94b0d48..6242b20 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -15,6 +15,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/test" ) @@ -172,10 +173,7 @@ func TestMailState(t *testing.T) { {"RCPT TO: u4@gmail.com", 250}, {"RSET", 250}, {"MAIL FROM:", 250}, - {"RCPT TO:name@host.com>", 250}, - {"RCPT TO:<\"user>name\"@host.com>", 250}, + {`RCPT TO:<"first/last"@host.com`, 250}, } if err := playSession(t, server, script); err != nil { t.Error(err) @@ -360,7 +358,8 @@ func setupSMTPServer(ds storage.Store) (s *Server, buf *bytes.Buffer, teardown f close(shutdownChan) cancel() } - s = NewServer(cfg, shutdownChan, ds, msghub.New(ctx, 100)) + apolicy := &policy.Addressing{Config: cfg} + s = NewServer(cfg, shutdownChan, ds, apolicy, msghub.New(ctx, 100)) return s, buf, teardown } diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index 83d5697..0e795b7 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -13,6 +13,7 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" ) @@ -48,9 +49,10 @@ type Server struct { storeMessages bool // Dependencies - dataStore storage.Store // Mailbox/message store - globalShutdown chan bool // Shuts down Inbucket - msgHub *msghub.Hub // Pub/sub for message info + dataStore storage.Store // Mailbox/message store + apolicy *policy.Addressing // Address policy + globalShutdown chan bool // Shuts down Inbucket + msgHub *msghub.Hub // Pub/sub for message info // State listener net.Listener // Incoming network connections @@ -83,6 +85,7 @@ func NewServer( cfg config.SMTPConfig, globalShutdown chan bool, ds storage.Store, + apolicy *policy.Addressing, msgHub *msghub.Hub) *Server { return &Server{ host: fmt.Sprintf("%v:%v", cfg.IP4address, cfg.IP4port), @@ -94,6 +97,7 @@ func NewServer( storeMessages: cfg.StoreMessages, globalShutdown: globalShutdown, dataStore: ds, + apolicy: apolicy, msgHub: msgHub, waitgroup: new(sync.WaitGroup), } From e84b1f89520a92d3f906143a2c440cda4629c621 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 14:02:50 -0700 Subject: [PATCH 18/26] storage: Make locking an implementation detail for #69 - file: Store handles its own locking #77 - file: Move mbox into its own file - file & test: remove LockFor() --- pkg/server/smtp/handler.go | 32 +--- pkg/storage/file/fstore.go | 296 +++++-------------------------------- pkg/storage/file/mbox.go | 242 ++++++++++++++++++++++++++++++ pkg/storage/storage.go | 3 - pkg/test/storage.go | 7 - 5 files changed, 288 insertions(+), 292 deletions(-) create mode 100644 pkg/storage/file/mbox.go diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index f15656b..1848fe2 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -368,23 +368,10 @@ func (ss *Session) dataHandler() { return } if bytes.Equal(lineBuf, []byte(".\r\n")) || bytes.Equal(lineBuf, []byte(".\n")) { - // Mail data complete + // Mail data complete. for _, recip := range ss.recipients { if recip.ShouldStore() { - // TODO temporary hack to fix #77 until datastore revamp - mu, err := ss.server.dataStore.LockFor(recip.LocalPart) - if err != nil { - ss.logError("Failed to get lock for %q: %s", recip.LocalPart, err) - // Delivery failure - ss.send(fmt.Sprintf("451 Failed to store message for %v", recip.LocalPart)) - ss.reset() - return - } - mu.Lock() - ok := ss.deliverMessage(recip, msgBuf.Bytes()) - mu.Unlock() - if !ok { - // Delivery failure + if ok := ss.deliverMessage(recip, msgBuf.Bytes()); !ok { ss.send(fmt.Sprintf("451 Failed to store message for %v", recip.LocalPart)) ss.reset() return @@ -397,28 +384,22 @@ func (ss *Session) dataHandler() { ss.reset() return } - // SMTP RFC says remove leading periods from input + // RFC says remove leading periods from input. if len(lineBuf) > 0 && lineBuf[0] == '.' { lineBuf = lineBuf[1:] } msgBuf.Write(lineBuf) if msgBuf.Len() > ss.server.maxMessageBytes { - // Max message size exceeded ss.send("552 Maximum message size exceeded") ss.logWarn("Max message size exceeded while in DATA") ss.reset() return } - } // end for + } } // deliverMessage creates and populates a new Message for the specified recipient func (ss *Session) deliverMessage(recip *policy.Recipient, content []byte) (ok bool) { - name, err := policy.ParseMailboxName(recip.LocalPart) - if err != nil { - // This parse already succeeded when MailboxFor was called, shouldn't fail here. - return false - } // TODO replace with something that only reads header? env, err := enmime.ReadEnvelope(bytes.NewReader(content)) if err != nil { @@ -441,7 +422,7 @@ func (ss *Session) deliverMessage(recip *policy.Recipient, content []byte) (ok b ss.remoteDomain, ss.remoteHost, ss.server.domain, recip.Address, stamp)) delivery := &message.Delivery{ Meta: message.Metadata{ - Mailbox: name, + Mailbox: recip.Mailbox, From: from[0], To: to, Date: time.Now(), @@ -455,8 +436,9 @@ func (ss *Session) deliverMessage(recip *policy.Recipient, content []byte) (ok b return false } // Broadcast message information. + // TODO this belongs in message pkg. broadcast := msghub.Message{ - Mailbox: name, + Mailbox: recip.Mailbox, ID: id, From: delivery.From().String(), To: stringutil.StringAddressList(delivery.To()), diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index dc20b5d..f6a60ec 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -2,7 +2,6 @@ package file import ( "bufio" - "encoding/gob" "fmt" "io" "io/ioutil" @@ -77,11 +76,13 @@ func New(cfg config.DataStoreConfig) storage.Store { // AddMessage adds a message to the specified mailbox. func (fs *Store) AddMessage(m storage.StoreMessage) (id string, err error) { - r, err := m.RawReader() + mb, err := fs.mbox(m.Mailbox()) if err != nil { return "", err } - mb, err := fs.mbox(m.Mailbox()) + mb.Lock() + defer mb.Unlock() + r, err := m.RawReader() if err != nil { return "", err } @@ -140,6 +141,8 @@ func (fs *Store) GetMessage(mailbox, id string) (storage.StoreMessage, error) { if err != nil { return nil, err } + mb.RLock() + defer mb.RUnlock() return mb.getMessage(id) } @@ -149,6 +152,8 @@ func (fs *Store) GetMessages(mailbox string) ([]storage.StoreMessage, error) { if err != nil { return nil, err } + mb.RLock() + defer mb.RUnlock() return mb.getMessages() } @@ -158,6 +163,8 @@ func (fs *Store) RemoveMessage(mailbox, id string) error { if err != nil { return err } + mb.Lock() + defer mb.Unlock() return mb.removeMessage(id) } @@ -167,6 +174,8 @@ func (fs *Store) PurgeMessages(mailbox string) error { if err != nil { return err } + mb.Lock() + defer mb.Unlock() return mb.purge() } @@ -196,12 +205,10 @@ func (fs *Store) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) erro // Loop over mailboxes for _, inf3 := range infos3 { if inf3.IsDir() { - mbdir := inf3.Name() - mbpath := filepath.Join(fs.mailPath, l1, l2, mbdir) - idx := filepath.Join(mbpath, indexFileName) - mb := &mbox{store: fs, dirName: mbdir, path: mbpath, - indexPath: idx} + mb := fs.mboxFromHash(inf3.Name()) + mb.RLock() msgs, err := mb.getMessages() + mb.RUnlock() if err != nil { return err } @@ -217,265 +224,40 @@ func (fs *Store) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) erro return nil } -// LockFor returns the RWMutex for this mailbox, or an error. -func (fs *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { - name, err := policy.ParseMailboxName(emailAddress) - if err != nil { - return nil, err - } - hash := stringutil.HashMailboxName(name) - return fs.hashLock.Get(hash), nil -} - -// NewMessage is temproary until #69 MessageData refactor -func (fs *Store) NewMessage(mailbox string) (storage.StoreMessage, error) { - mb, err := fs.mbox(mailbox) - if err != nil { - return nil, err - } - return mb.newMessage() -} - // mbox returns the named mailbox. func (fs *Store) mbox(mailbox string) (*mbox, error) { name, err := policy.ParseMailboxName(mailbox) if err != nil { return nil, err } - dir := stringutil.HashMailboxName(name) - s1 := dir[0:3] - s2 := dir[0:6] - path := filepath.Join(fs.mailPath, s1, s2, dir) + hash := stringutil.HashMailboxName(name) + s1 := hash[0:3] + s2 := hash[0:6] + path := filepath.Join(fs.mailPath, s1, s2, hash) indexPath := filepath.Join(path, indexFileName) - - return &mbox{store: fs, name: name, dirName: dir, path: path, - indexPath: indexPath}, nil + return &mbox{ + RWMutex: fs.hashLock.Get(hash), + store: fs, + name: name, + dirName: hash, + path: path, + indexPath: indexPath, + }, nil } -// mbox manages the mail for a specific user and correlates to a particular directory on disk. -type mbox struct { - store *Store - name string - dirName string - path string - indexLoaded bool - indexPath string - messages []*Message -} - -// getMessages scans the mailbox directory for .gob files and decodes them into -// a slice of Message objects. -func (mb *mbox) getMessages() ([]storage.StoreMessage, error) { - if !mb.indexLoaded { - if err := mb.readIndex(); err != nil { - return nil, err - } +// mboxFromPath constructs a mailbox based on name hash. +func (fs *Store) mboxFromHash(hash string) *mbox { + s1 := hash[0:3] + s2 := hash[0:6] + path := filepath.Join(fs.mailPath, s1, s2, hash) + indexPath := filepath.Join(path, indexFileName) + return &mbox{ + RWMutex: fs.hashLock.Get(hash), + store: fs, + dirName: hash, + path: path, + indexPath: indexPath, } - messages := make([]storage.StoreMessage, len(mb.messages)) - for i, m := range mb.messages { - messages[i] = m - } - return messages, nil -} - -// getMessage decodes a single message by ID and returns a Message object. -func (mb *mbox) getMessage(id string) (storage.StoreMessage, error) { - if !mb.indexLoaded { - if err := mb.readIndex(); err != nil { - return nil, err - } - } - if id == "latest" && len(mb.messages) != 0 { - return mb.messages[len(mb.messages)-1], nil - } - for _, m := range mb.messages { - if m.Fid == id { - return m, nil - } - } - return nil, storage.ErrNotExist -} - -// removeMessage deletes the message off disk and removes it from the index. -func (mb *mbox) removeMessage(id string) error { - if !mb.indexLoaded { - if err := mb.readIndex(); err != nil { - return err - } - } - var msg *Message - for i, m := range mb.messages { - if id == m.ID() { - msg = m - // Slice around message we are deleting - mb.messages = append(mb.messages[:i], mb.messages[i+1:]...) - break - } - } - if msg == nil { - return storage.ErrNotExist - } - if err := mb.writeIndex(); err != nil { - return err - } - if len(mb.messages) == 0 { - // This was the last message, thus writeIndex() has removed the entire - // directory; we don't need to delete the raw file. - return nil - } - // There are still messages in the index - log.Tracef("Deleting %v", msg.rawPath()) - return os.Remove(msg.rawPath()) -} - -// purge deletes all messages in this mailbox. -func (mb *mbox) purge() error { - mb.messages = mb.messages[:0] - return mb.writeIndex() -} - -// readIndex loads the mailbox index data from disk -func (mb *mbox) readIndex() error { - // Clear message slice, open index - mb.messages = mb.messages[:0] - // Lock for reading - indexMx.RLock() - defer indexMx.RUnlock() - // Check if index exists - if _, err := os.Stat(mb.indexPath); err != nil { - // Does not exist, but that's not an error in our world - log.Tracef("Index %v does not exist (yet)", mb.indexPath) - mb.indexLoaded = true - return nil - } - file, err := os.Open(mb.indexPath) - if err != nil { - return err - } - defer func() { - if err := file.Close(); err != nil { - log.Errorf("Failed to close %q: %v", mb.indexPath, err) - } - }() - // Decode gob data - dec := gob.NewDecoder(bufio.NewReader(file)) - name := "" - if err = dec.Decode(&name); err != nil { - return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) - } - mb.name = name - for { - // Load messages until EOF - msg := &Message{} - if err = dec.Decode(msg); err != nil { - if err == io.EOF { - break - } - return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) - } - msg.mailbox = mb - mb.messages = append(mb.messages, msg) - } - mb.indexLoaded = true - return nil -} - -// writeIndex overwrites the index on disk with the current mailbox data -func (mb *mbox) writeIndex() error { - // Lock for writing - indexMx.Lock() - defer indexMx.Unlock() - if len(mb.messages) > 0 { - // Ensure mailbox directory exists - if err := mb.createDir(); err != nil { - return err - } - // Open index for writing - file, err := os.Create(mb.indexPath) - if err != nil { - return err - } - writer := bufio.NewWriter(file) - // Write each message and then flush - enc := gob.NewEncoder(writer) - if err = enc.Encode(mb.name); err != nil { - _ = file.Close() - return err - } - for _, m := range mb.messages { - if err = enc.Encode(m); err != nil { - _ = file.Close() - return err - } - } - if err := writer.Flush(); err != nil { - _ = file.Close() - return err - } - if err := file.Close(); err != nil { - log.Errorf("Failed to close %q: %v", mb.indexPath, err) - return err - } - } else { - // No messages, delete index+maildir - log.Tracef("Removing mailbox %v", mb.path) - return mb.removeDir() - } - return nil -} - -// createDir checks for the presence of the path for this mailbox, creates it if needed -func (mb *mbox) createDir() error { - dirMx.Lock() - defer dirMx.Unlock() - if _, err := os.Stat(mb.path); err != nil { - if err := os.MkdirAll(mb.path, 0770); err != nil { - log.Errorf("Failed to create directory %v, %v", mb.path, err) - return err - } - } - return nil -} - -// removeDir removes the mailbox, plus empty higher level directories -func (mb *mbox) removeDir() error { - dirMx.Lock() - defer dirMx.Unlock() - // remove mailbox dir, including index file - if err := os.RemoveAll(mb.path); err != nil { - return err - } - // remove parents if empty - dir := filepath.Dir(mb.path) - if removeDirIfEmpty(dir) { - removeDirIfEmpty(filepath.Dir(dir)) - } - return nil -} - -// removeDirIfEmpty will remove the specified directory if it contains no files or directories. -// Caller should hold dirMx. Returns true if dir was removed. -func removeDirIfEmpty(path string) (removed bool) { - f, err := os.Open(path) - if err != nil { - return false - } - files, err := f.Readdirnames(0) - _ = f.Close() - if err != nil { - return false - } - if len(files) > 0 { - // Dir not empty - return false - } - log.Tracef("Removing dir %v", path) - err = os.Remove(path) - if err != nil { - log.Errorf("Failed to remove %q: %v", path, err) - return false - } - return true } // generatePrefix converts a Time object into the ISO style format we use diff --git a/pkg/storage/file/mbox.go b/pkg/storage/file/mbox.go new file mode 100644 index 0000000..ea6d7f4 --- /dev/null +++ b/pkg/storage/file/mbox.go @@ -0,0 +1,242 @@ +package file + +import ( + "bufio" + "encoding/gob" + "fmt" + "io" + "os" + "path/filepath" + "sync" + + "github.com/jhillyerd/inbucket/pkg/log" + "github.com/jhillyerd/inbucket/pkg/storage" +) + +// mbox manages the mail for a specific user and correlates to a particular directory on disk. +// mbox methods are not thread safe, mbox.RWMutex must be held prior to calling. +type mbox struct { + *sync.RWMutex + store *Store + name string + dirName string + path string + indexLoaded bool + indexPath string + messages []*Message +} + +// getMessages scans the mailbox directory for .gob files and decodes them into +// a slice of Message objects. +func (mb *mbox) getMessages() ([]storage.StoreMessage, error) { + if !mb.indexLoaded { + if err := mb.readIndex(); err != nil { + return nil, err + } + } + messages := make([]storage.StoreMessage, len(mb.messages)) + for i, m := range mb.messages { + messages[i] = m + } + return messages, nil +} + +// getMessage decodes a single message by ID and returns a Message object. +func (mb *mbox) getMessage(id string) (storage.StoreMessage, error) { + if !mb.indexLoaded { + if err := mb.readIndex(); err != nil { + return nil, err + } + } + if id == "latest" && len(mb.messages) != 0 { + return mb.messages[len(mb.messages)-1], nil + } + for _, m := range mb.messages { + if m.Fid == id { + return m, nil + } + } + return nil, storage.ErrNotExist +} + +// removeMessage deletes the message off disk and removes it from the index. +func (mb *mbox) removeMessage(id string) error { + if !mb.indexLoaded { + if err := mb.readIndex(); err != nil { + return err + } + } + var msg *Message + for i, m := range mb.messages { + if id == m.ID() { + msg = m + // Slice around message we are deleting + mb.messages = append(mb.messages[:i], mb.messages[i+1:]...) + break + } + } + if msg == nil { + return storage.ErrNotExist + } + if err := mb.writeIndex(); err != nil { + return err + } + if len(mb.messages) == 0 { + // This was the last message, thus writeIndex() has removed the entire + // directory; we don't need to delete the raw file. + return nil + } + // There are still messages in the index + log.Tracef("Deleting %v", msg.rawPath()) + return os.Remove(msg.rawPath()) +} + +// purge deletes all messages in this mailbox. +func (mb *mbox) purge() error { + mb.messages = mb.messages[:0] + return mb.writeIndex() +} + +// readIndex loads the mailbox index data from disk +func (mb *mbox) readIndex() error { + // Clear message slice, open index + mb.messages = mb.messages[:0] + // Lock for reading + indexMx.RLock() + defer indexMx.RUnlock() + // Check if index exists + if _, err := os.Stat(mb.indexPath); err != nil { + // Does not exist, but that's not an error in our world + log.Tracef("Index %v does not exist (yet)", mb.indexPath) + mb.indexLoaded = true + return nil + } + file, err := os.Open(mb.indexPath) + if err != nil { + return err + } + defer func() { + if err := file.Close(); err != nil { + log.Errorf("Failed to close %q: %v", mb.indexPath, err) + } + }() + // Decode gob data + dec := gob.NewDecoder(bufio.NewReader(file)) + name := "" + if err = dec.Decode(&name); err != nil { + return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) + } + mb.name = name + for { + // Load messages until EOF + msg := &Message{} + if err = dec.Decode(msg); err != nil { + if err == io.EOF { + break + } + return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) + } + msg.mailbox = mb + mb.messages = append(mb.messages, msg) + } + mb.indexLoaded = true + return nil +} + +// writeIndex overwrites the index on disk with the current mailbox data +func (mb *mbox) writeIndex() error { + // Lock for writing + indexMx.Lock() + defer indexMx.Unlock() + if len(mb.messages) > 0 { + // Ensure mailbox directory exists + if err := mb.createDir(); err != nil { + return err + } + // Open index for writing + file, err := os.Create(mb.indexPath) + if err != nil { + return err + } + writer := bufio.NewWriter(file) + // Write each message and then flush + enc := gob.NewEncoder(writer) + if err = enc.Encode(mb.name); err != nil { + _ = file.Close() + return err + } + for _, m := range mb.messages { + if err = enc.Encode(m); err != nil { + _ = file.Close() + return err + } + } + if err := writer.Flush(); err != nil { + _ = file.Close() + return err + } + if err := file.Close(); err != nil { + log.Errorf("Failed to close %q: %v", mb.indexPath, err) + return err + } + } else { + // No messages, delete index+maildir + log.Tracef("Removing mailbox %v", mb.path) + return mb.removeDir() + } + return nil +} + +// createDir checks for the presence of the path for this mailbox, creates it if needed +func (mb *mbox) createDir() error { + dirMx.Lock() + defer dirMx.Unlock() + if _, err := os.Stat(mb.path); err != nil { + if err := os.MkdirAll(mb.path, 0770); err != nil { + log.Errorf("Failed to create directory %v, %v", mb.path, err) + return err + } + } + return nil +} + +// removeDir removes the mailbox, plus empty higher level directories +func (mb *mbox) removeDir() error { + dirMx.Lock() + defer dirMx.Unlock() + // remove mailbox dir, including index file + if err := os.RemoveAll(mb.path); err != nil { + return err + } + // remove parents if empty + dir := filepath.Dir(mb.path) + if removeDirIfEmpty(dir) { + removeDirIfEmpty(filepath.Dir(dir)) + } + return nil +} + +// removeDirIfEmpty will remove the specified directory if it contains no files or directories. +// Caller should hold dirMx. Returns true if dir was removed. +func removeDirIfEmpty(path string) (removed bool) { + f, err := os.Open(path) + if err != nil { + return false + } + files, err := f.Readdirnames(0) + _ = f.Close() + if err != nil { + return false + } + if len(files) > 0 { + // Dir not empty + return false + } + log.Tracef("Removing dir %v", path) + err = os.Remove(path) + if err != nil { + log.Errorf("Failed to remove %q: %v", path, err) + return false + } + return true +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 62d4972..f8bcaef 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -5,7 +5,6 @@ import ( "errors" "io" "net/mail" - "sync" "time" ) @@ -26,8 +25,6 @@ type Store interface { PurgeMessages(mailbox string) error RemoveMessage(mailbox, id string) error VisitMailboxes(f func([]StoreMessage) (cont bool)) error - // LockFor is a temporary hack to fix #77 until Datastore revamp - LockFor(emailAddress string) (*sync.RWMutex, error) } // StoreMessage represents a message to be stored, or returned from a storage implementation. diff --git a/pkg/test/storage.go b/pkg/test/storage.go index 52c9b00..2c45b10 100644 --- a/pkg/test/storage.go +++ b/pkg/test/storage.go @@ -2,7 +2,6 @@ package test import ( "errors" - "sync" "github.com/jhillyerd/inbucket/pkg/storage" ) @@ -82,12 +81,6 @@ func (s *StoreStub) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) e return nil } -// LockFor mock function returns a new RWMutex, never errors. -// TODO(#69) remove -func (s *StoreStub) LockFor(name string) (*sync.RWMutex, error) { - return &sync.RWMutex{}, nil -} - // MessageDeleted returns true if the specified message was deleted func (s *StoreStub) MessageDeleted(m storage.StoreMessage) bool { _, ok := s.deleted[m] From dc4db59211e142b3195914cdcb8f7e8d2a0a6eee Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 14:41:03 -0700 Subject: [PATCH 19/26] smtp: Don't require MIME headers for metadata This was a regression, will again fall back to MAIL FROM/RCPT TO data. --- pkg/server/smtp/handler.go | 10 ++++++---- pkg/server/smtp/handler_test.go | 30 ++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 1848fe2..686f94a 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net" + "net/mail" "regexp" "strconv" "strings" @@ -408,13 +409,14 @@ func (ss *Session) deliverMessage(recip *policy.Recipient, content []byte) (ok b } from, err := env.AddressList("From") if err != nil { - ss.logError("Failed to get From address: %v", err) - return false + from = []*mail.Address{{Address: ss.from}} } to, err := env.AddressList("To") if err != nil { - ss.logError("Failed to get To addresses: %v", err) - return false + to = make([]*mail.Address, len(ss.recipients)) + for i, torecip := range ss.recipients { + to[i] = &torecip.Address + } } // Generate Received header. stamp := time.Now().Format(timeStampFormat) diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index 6242b20..49100d2 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -200,7 +200,7 @@ func TestMailState(t *testing.T) { {"MAIL FROM:", 250}, {"RCPT TO:", 250}, {"DATA", 354}, - {".", 451}, + {".", 250}, } if err := playSession(t, server, script); err != nil { t.Error(err) @@ -247,7 +247,6 @@ func TestDataState(t *testing.T) { pipe := setupSMTPSession(server) c := textproto.NewConn(pipe) - // Get us into DATA state if code, _, err := c.ReadCodeLine(220); err != nil { t.Errorf("Expected a 220 greeting, got %v", code) } @@ -274,6 +273,33 @@ Hi! t.Errorf("Expected a 250 greeting, got %v", code) } + // Test with no useful headers. + pipe = setupSMTPSession(server) + c = textproto.NewConn(pipe) + if code, _, err := c.ReadCodeLine(220); err != nil { + t.Errorf("Expected a 220 greeting, got %v", code) + } + script = []scriptStep{ + {"HELO localhost", 250}, + {"MAIL FROM:", 250}, + {"RCPT TO:", 250}, + {"DATA", 354}, + } + if err := playScriptAgainst(t, c, script); err != nil { + t.Error(err) + } + // Send a message + body = `X-Useless-Header: true + +Hi! Can you still deliver this? +` + dw = c.DotWriter() + _, _ = io.WriteString(dw, body) + _ = dw.Close() + if code, _, err := c.ReadCodeLine(250); err != nil { + t.Errorf("Expected a 250 greeting, got %v", code) + } + if t.Failed() { // Wait for handler to finish logging time.Sleep(2 * time.Second) From a22412f65e737cb0dc182098990a07a1fd9b8503 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 15:17:44 -0700 Subject: [PATCH 20/26] manager: Add MailboxForAddress(), calls policy pkg #84 --- pkg/message/manager.go | 7 +++++++ pkg/rest/apiv1_controller.go | 11 +++++------ pkg/rest/socketv1_controller.go | 3 +-- pkg/test/manager.go | 6 ++++++ pkg/webui/mailbox_controller.go | 17 ++++++++--------- pkg/webui/root_controller.go | 3 +-- 6 files changed, 28 insertions(+), 19 deletions(-) diff --git a/pkg/message/manager.go b/pkg/message/manager.go index 5782273..1c4e04e 100644 --- a/pkg/message/manager.go +++ b/pkg/message/manager.go @@ -4,6 +4,7 @@ import ( "io" "github.com/jhillyerd/enmime" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" ) @@ -14,6 +15,7 @@ type Manager interface { PurgeMessages(mailbox string) error RemoveMessage(mailbox, id string) error SourceReader(mailbox, id string) (io.ReadCloser, error) + MailboxForAddress(address string) (string, error) } // StoreManager is a message Manager backed by the storage.Store. @@ -72,6 +74,11 @@ func (s *StoreManager) SourceReader(mailbox, id string) (io.ReadCloser, error) { return sm.RawReader() } +// MailboxForAddress parses an email address to return the canonical mailbox name. +func (s *StoreManager) MailboxForAddress(mailbox string) (string, error) { + return policy.ParseMailboxName(mailbox) +} + // makeMetadata populates Metadata from a StoreMessage. func makeMetadata(m storage.StoreMessage) *Metadata { return &Metadata{ diff --git a/pkg/rest/apiv1_controller.go b/pkg/rest/apiv1_controller.go index 2b10eb0..c463ecf 100644 --- a/pkg/rest/apiv1_controller.go +++ b/pkg/rest/apiv1_controller.go @@ -11,7 +11,6 @@ import ( "strconv" "github.com/jhillyerd/inbucket/pkg/log" - "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/rest/model" "github.com/jhillyerd/inbucket/pkg/server/web" "github.com/jhillyerd/inbucket/pkg/storage" @@ -21,7 +20,7 @@ import ( // MailboxListV1 renders a list of messages in a mailbox func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -51,7 +50,7 @@ func MailboxListV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -101,7 +100,7 @@ func MailboxShowV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( // MailboxPurgeV1 deletes all messages from a mailbox func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -119,7 +118,7 @@ func MailboxPurgeV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -143,7 +142,7 @@ func MailboxSourceV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) func MailboxDeleteV1(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } diff --git a/pkg/rest/socketv1_controller.go b/pkg/rest/socketv1_controller.go index 7614ac1..d0ceddd 100644 --- a/pkg/rest/socketv1_controller.go +++ b/pkg/rest/socketv1_controller.go @@ -7,7 +7,6 @@ import ( "github.com/gorilla/websocket" "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/msghub" - "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/rest/model" "github.com/jhillyerd/inbucket/pkg/server/web" ) @@ -173,7 +172,7 @@ func MonitorAllMessagesV1( // notifies the client of messages received by a particular mailbox. func MonitorMailboxMessagesV1( w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } diff --git a/pkg/test/manager.go b/pkg/test/manager.go index 47c87a8..bf8d9ad 100644 --- a/pkg/test/manager.go +++ b/pkg/test/manager.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/jhillyerd/inbucket/pkg/message" + "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" ) @@ -51,3 +52,8 @@ func (m *ManagerStub) GetMetadata(mailbox string) ([]*message.Metadata, error) { } return metas, nil } + +// MailboxForAddress invokes policy.ParseMailboxName. +func (m *ManagerStub) MailboxForAddress(address string) (string, error) { + return policy.ParseMailboxName(address) +} diff --git a/pkg/webui/mailbox_controller.go b/pkg/webui/mailbox_controller.go index 2af9ef1..d876b3a 100644 --- a/pkg/webui/mailbox_controller.go +++ b/pkg/webui/mailbox_controller.go @@ -8,7 +8,6 @@ import ( "strconv" "github.com/jhillyerd/inbucket/pkg/log" - "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/server/web" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/webui/sanitize" @@ -25,7 +24,7 @@ func MailboxIndex(w http.ResponseWriter, req *http.Request, ctx *web.Context) (e http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - name, err = policy.ParseMailboxName(name) + name, err = ctx.Manager.MailboxForAddress(name) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) @@ -52,7 +51,7 @@ func MailboxIndex(w http.ResponseWriter, req *http.Request, ctx *web.Context) (e func MailboxLink(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) @@ -68,7 +67,7 @@ func MailboxLink(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er // MailboxList renders a list of messages in a mailbox. Renders a partial func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -90,7 +89,7 @@ func MailboxList(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -131,7 +130,7 @@ func MailboxShow(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -159,7 +158,7 @@ func MailboxHTML(w http.ResponseWriter, req *http.Request, ctx *web.Context) (er func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { return err } @@ -183,7 +182,7 @@ func MailboxSource(w http.ResponseWriter, req *http.Request, ctx *web.Context) ( func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 id := ctx.Vars["id"] - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) @@ -225,7 +224,7 @@ func MailboxDownloadAttach(w http.ResponseWriter, req *http.Request, ctx *web.Co // MailboxViewAttach sends the attachment to the client for online viewing func MailboxViewAttach(w http.ResponseWriter, req *http.Request, ctx *web.Context) (err error) { // Don't have to validate these aren't empty, Gorilla returns 404 - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) diff --git a/pkg/webui/root_controller.go b/pkg/webui/root_controller.go index 21d4dd3..0ea5780 100644 --- a/pkg/webui/root_controller.go +++ b/pkg/webui/root_controller.go @@ -7,7 +7,6 @@ import ( "net/http" "github.com/jhillyerd/inbucket/pkg/config" - "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/server/web" ) @@ -58,7 +57,7 @@ func RootMonitorMailbox(w http.ResponseWriter, req *http.Request, ctx *web.Conte http.Redirect(w, req, web.Reverse("RootIndex"), http.StatusSeeOther) return nil } - name, err := policy.ParseMailboxName(ctx.Vars["name"]) + name, err := ctx.Manager.MailboxForAddress(ctx.Vars["name"]) if err != nil { ctx.Session.AddFlash(err.Error(), "errors") _ = ctx.Session.Save(req, w) From f953bcf4bbb7c9281f3b93f7323e74be8da4f500 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 16:54:29 -0700 Subject: [PATCH 21/26] smtp: Move delivery into message.Manager for #69 --- cmd/inbucket/main.go | 31 ++++++-------- pkg/message/manager.go | 68 +++++++++++++++++++++++++++++++ pkg/server/smtp/handler.go | 71 +++++---------------------------- pkg/server/smtp/handler_test.go | 8 ++-- pkg/server/smtp/listener.go | 17 ++++---- 5 files changed, 102 insertions(+), 93 deletions(-) diff --git a/cmd/inbucket/main.go b/cmd/inbucket/main.go index ff5a313..0de0b07 100644 --- a/cmd/inbucket/main.go +++ b/cmd/inbucket/main.go @@ -115,32 +115,27 @@ func main() { } } - // Create message hub + // Configure internal services. msgHub := msghub.New(rootCtx, config.GetWebConfig().MonitorHistory) - - // Setup our datastore dscfg := config.GetDataStoreConfig() - ds := file.New(dscfg) - retentionScanner := storage.NewRetentionScanner(dscfg, ds, shutdownChan) + store := file.New(dscfg) + apolicy := &policy.Addressing{Config: config.GetSMTPConfig()} + mmanager := &message.StoreManager{Store: store, Hub: msgHub} + // Start Retention scanner. + retentionScanner := storage.NewRetentionScanner(dscfg, store, shutdownChan) retentionScanner.Start() - - // Start HTTP server - mm := &message.StoreManager{Store: ds} - web.Initialize(config.GetWebConfig(), shutdownChan, mm, msgHub) + // Start HTTP server. + web.Initialize(config.GetWebConfig(), shutdownChan, mmanager, msgHub) webui.SetupRoutes(web.Router) rest.SetupRoutes(web.Router) go web.Start(rootCtx) - - // Start POP3 server - pop3Server = pop3.New(config.GetPOP3Config(), shutdownChan, ds) + // Start POP3 server. + pop3Server = pop3.New(config.GetPOP3Config(), shutdownChan, store) go pop3Server.Start(rootCtx) - - // Startup SMTP server - apolicy := &policy.Addressing{Config: config.GetSMTPConfig()} - smtpServer = smtp.NewServer(config.GetSMTPConfig(), shutdownChan, ds, apolicy, msgHub) + // Start SMTP server. + smtpServer = smtp.NewServer(config.GetSMTPConfig(), shutdownChan, mmanager, apolicy) go smtpServer.Start(rootCtx) - - // Loop forever waiting for signals or shutdown channel + // Loop forever waiting for signals or shutdown channel. signalLoop: for { select { diff --git a/pkg/message/manager.go b/pkg/message/manager.go index 1c4e04e..c9b74f1 100644 --- a/pkg/message/manager.go +++ b/pkg/message/manager.go @@ -1,15 +1,28 @@ package message import ( + "bytes" "io" + "net/mail" + "strings" + "time" "github.com/jhillyerd/enmime" + "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" + "github.com/jhillyerd/inbucket/pkg/stringutil" ) // Manager is the interface controllers use to interact with messages. type Manager interface { + Deliver( + to *policy.Recipient, + from string, + recipients []*policy.Recipient, + prefix string, + content []byte, + ) (id string, err error) GetMetadata(mailbox string) ([]*Metadata, error) GetMessage(mailbox, id string) (*Message, error) PurgeMessages(mailbox string) error @@ -21,6 +34,61 @@ type Manager interface { // StoreManager is a message Manager backed by the storage.Store. type StoreManager struct { Store storage.Store + Hub *msghub.Hub +} + +// Deliver submits a new message to the store. +func (s *StoreManager) Deliver( + to *policy.Recipient, + from string, + recipients []*policy.Recipient, + prefix string, + content []byte, +) (string, error) { + // TODO enmime is too heavy for this step, only need header + env, err := enmime.ReadEnvelope(bytes.NewReader(content)) + if err != nil { + return "", err + } + fromaddr, err := env.AddressList("From") + if err != nil || len(fromaddr) == 0 { + fromaddr = []*mail.Address{{Address: from}} + } + toaddr, err := env.AddressList("To") + if err != nil { + toaddr = make([]*mail.Address, len(recipients)) + for i, torecip := range recipients { + toaddr[i] = &torecip.Address + } + } + delivery := &Delivery{ + Meta: Metadata{ + Mailbox: to.Mailbox, + From: fromaddr[0], + To: toaddr, + Date: time.Now(), + Subject: env.GetHeader("Subject"), + }, + Reader: io.MultiReader(strings.NewReader(prefix), bytes.NewReader(content)), + } + id, err := s.Store.AddMessage(delivery) + if err != nil { + return "", err + } + if s.Hub != nil { + // Broadcast message information. + broadcast := msghub.Message{ + Mailbox: to.Mailbox, + ID: id, + From: delivery.From().String(), + To: stringutil.StringAddressList(delivery.To()), + Subject: delivery.Subject(), + Date: delivery.Date(), + Size: delivery.Size(), + } + s.Hub.Dispatch(broadcast) + } + return id, nil } // GetMetadata returns a slice of metadata for the specified mailbox. diff --git a/pkg/server/smtp/handler.go b/pkg/server/smtp/handler.go index 686f94a..4d6ee33 100644 --- a/pkg/server/smtp/handler.go +++ b/pkg/server/smtp/handler.go @@ -6,18 +6,13 @@ import ( "fmt" "io" "net" - "net/mail" "regexp" "strconv" "strings" "time" - "github.com/jhillyerd/enmime" "github.com/jhillyerd/inbucket/pkg/log" - "github.com/jhillyerd/inbucket/pkg/message" - "github.com/jhillyerd/inbucket/pkg/msghub" "github.com/jhillyerd/inbucket/pkg/policy" - "github.com/jhillyerd/inbucket/pkg/stringutil" ) // State tracks the current mode of our SMTP state machine @@ -370,9 +365,18 @@ func (ss *Session) dataHandler() { } if bytes.Equal(lineBuf, []byte(".\r\n")) || bytes.Equal(lineBuf, []byte(".\n")) { // Mail data complete. + tstamp := time.Now().Format(timeStampFormat) for _, recip := range ss.recipients { if recip.ShouldStore() { - if ok := ss.deliverMessage(recip, msgBuf.Bytes()); !ok { + // Generate Received header. + prefix := fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", + ss.remoteDomain, ss.remoteHost, ss.server.domain, recip.Address.Address, + tstamp) + // Deliver message. + _, err := ss.server.manager.Deliver( + recip, ss.from, ss.recipients, prefix, msgBuf.Bytes()) + if err != nil { + ss.logError("delivery for %v: %v", recip.LocalPart, err) ss.send(fmt.Sprintf("451 Failed to store message for %v", recip.LocalPart)) ss.reset() return @@ -385,7 +389,7 @@ func (ss *Session) dataHandler() { ss.reset() return } - // RFC says remove leading periods from input. + // RFC: remove leading periods from DATA. if len(lineBuf) > 0 && lineBuf[0] == '.' { lineBuf = lineBuf[1:] } @@ -399,59 +403,6 @@ func (ss *Session) dataHandler() { } } -// deliverMessage creates and populates a new Message for the specified recipient -func (ss *Session) deliverMessage(recip *policy.Recipient, content []byte) (ok bool) { - // TODO replace with something that only reads header? - env, err := enmime.ReadEnvelope(bytes.NewReader(content)) - if err != nil { - ss.logError("Failed to parse message for %q: %v", recip.LocalPart, err) - return false - } - from, err := env.AddressList("From") - if err != nil { - from = []*mail.Address{{Address: ss.from}} - } - to, err := env.AddressList("To") - if err != nil { - to = make([]*mail.Address, len(ss.recipients)) - for i, torecip := range ss.recipients { - to[i] = &torecip.Address - } - } - // Generate Received header. - stamp := time.Now().Format(timeStampFormat) - recd := strings.NewReader(fmt.Sprintf("Received: from %s ([%s]) by %s\r\n for <%s>; %s\r\n", - ss.remoteDomain, ss.remoteHost, ss.server.domain, recip.Address, stamp)) - delivery := &message.Delivery{ - Meta: message.Metadata{ - Mailbox: recip.Mailbox, - From: from[0], - To: to, - Date: time.Now(), - Subject: env.GetHeader("Subject"), - }, - Reader: io.MultiReader(recd, bytes.NewReader(content)), - } - id, err := ss.server.dataStore.AddMessage(delivery) - if err != nil { - ss.logError("Failed to store message for %q: %s", recip.LocalPart, err) - return false - } - // Broadcast message information. - // TODO this belongs in message pkg. - broadcast := msghub.Message{ - Mailbox: recip.Mailbox, - ID: id, - From: delivery.From().String(), - To: stringutil.StringAddressList(delivery.To()), - Subject: delivery.Subject(), - Date: delivery.Date(), - Size: delivery.Size(), - } - ss.server.msgHub.Dispatch(broadcast) - return true -} - func (ss *Session) enterState(state State) { ss.state = state ss.logTrace("Entering state %v", state) diff --git a/pkg/server/smtp/handler_test.go b/pkg/server/smtp/handler_test.go index 49100d2..283c6d1 100644 --- a/pkg/server/smtp/handler_test.go +++ b/pkg/server/smtp/handler_test.go @@ -2,7 +2,6 @@ package smtp import ( "bytes" - "context" "fmt" "io" @@ -14,7 +13,7 @@ import ( "time" "github.com/jhillyerd/inbucket/pkg/config" - "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/policy" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/test" @@ -379,13 +378,12 @@ func setupSMTPServer(ds storage.Store) (s *Server, buf *bytes.Buffer, teardown f // Create a server, don't start it shutdownChan := make(chan bool) - ctx, cancel := context.WithCancel(context.Background()) teardown = func() { close(shutdownChan) - cancel() } apolicy := &policy.Addressing{Config: cfg} - s = NewServer(cfg, shutdownChan, ds, apolicy, msghub.New(ctx, 100)) + manager := &message.StoreManager{Store: ds} + s = NewServer(cfg, shutdownChan, manager, apolicy) return s, buf, teardown } diff --git a/pkg/server/smtp/listener.go b/pkg/server/smtp/listener.go index 0e795b7..959b6f5 100644 --- a/pkg/server/smtp/listener.go +++ b/pkg/server/smtp/listener.go @@ -12,9 +12,8 @@ import ( "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/log" - "github.com/jhillyerd/inbucket/pkg/msghub" + "github.com/jhillyerd/inbucket/pkg/message" "github.com/jhillyerd/inbucket/pkg/policy" - "github.com/jhillyerd/inbucket/pkg/storage" ) func init() { @@ -49,10 +48,9 @@ type Server struct { storeMessages bool // Dependencies - dataStore storage.Store // Mailbox/message store - apolicy *policy.Addressing // Address policy - globalShutdown chan bool // Shuts down Inbucket - msgHub *msghub.Hub // Pub/sub for message info + apolicy *policy.Addressing // Address policy. + globalShutdown chan bool // Shuts down Inbucket. + manager message.Manager // Used to deliver messages. // State listener net.Listener // Incoming network connections @@ -84,9 +82,9 @@ var ( func NewServer( cfg config.SMTPConfig, globalShutdown chan bool, - ds storage.Store, + manager message.Manager, apolicy *policy.Addressing, - msgHub *msghub.Hub) *Server { +) *Server { return &Server{ host: fmt.Sprintf("%v:%v", cfg.IP4address, cfg.IP4port), domain: cfg.Domain, @@ -96,9 +94,8 @@ func NewServer( maxMessageBytes: cfg.MaxMessageBytes, storeMessages: cfg.StoreMessages, globalShutdown: globalShutdown, - dataStore: ds, + manager: manager, apolicy: apolicy, - msgHub: msgHub, waitgroup: new(sync.WaitGroup), } } From 30a329c0d31a9be255f7083eeeca994708604d05 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 17 Mar 2018 17:56:06 -0700 Subject: [PATCH 22/26] Renames, closes #69 - storage: rename StoreMessage to Message - storage: rename Message.RawReader() to Source() --- pkg/message/manager.go | 14 +++++++------- pkg/message/message.go | 6 +++--- pkg/server/pop3/handler.go | 30 +++++++++++++++--------------- pkg/storage/file/fmessage.go | 4 ++-- pkg/storage/file/fstore.go | 10 +++++----- pkg/storage/file/fstore_test.go | 2 +- pkg/storage/file/mbox.go | 6 +++--- pkg/storage/retention.go | 2 +- pkg/storage/retention_test.go | 6 +++--- pkg/storage/storage.go | 14 +++++++------- pkg/test/storage.go | 20 ++++++++++---------- pkg/test/storage_suite.go | 6 +++--- 12 files changed, 60 insertions(+), 60 deletions(-) diff --git a/pkg/message/manager.go b/pkg/message/manager.go index c9b74f1..2a8bb47 100644 --- a/pkg/message/manager.go +++ b/pkg/message/manager.go @@ -43,10 +43,10 @@ func (s *StoreManager) Deliver( from string, recipients []*policy.Recipient, prefix string, - content []byte, + source []byte, ) (string, error) { // TODO enmime is too heavy for this step, only need header - env, err := enmime.ReadEnvelope(bytes.NewReader(content)) + env, err := enmime.ReadEnvelope(bytes.NewReader(source)) if err != nil { return "", err } @@ -69,7 +69,7 @@ func (s *StoreManager) Deliver( Date: time.Now(), Subject: env.GetHeader("Subject"), }, - Reader: io.MultiReader(strings.NewReader(prefix), bytes.NewReader(content)), + Reader: io.MultiReader(strings.NewReader(prefix), bytes.NewReader(source)), } id, err := s.Store.AddMessage(delivery) if err != nil { @@ -110,7 +110,7 @@ func (s *StoreManager) GetMessage(mailbox, id string) (*Message, error) { if err != nil { return nil, err } - r, err := sm.RawReader() + r, err := sm.Source() if err != nil { return nil, err } @@ -139,7 +139,7 @@ func (s *StoreManager) SourceReader(mailbox, id string) (io.ReadCloser, error) { if err != nil { return nil, err } - return sm.RawReader() + return sm.Source() } // MailboxForAddress parses an email address to return the canonical mailbox name. @@ -147,8 +147,8 @@ func (s *StoreManager) MailboxForAddress(mailbox string) (string, error) { return policy.ParseMailboxName(mailbox) } -// makeMetadata populates Metadata from a StoreMessage. -func makeMetadata(m storage.StoreMessage) *Metadata { +// makeMetadata populates Metadata from a storage.Message. +func makeMetadata(m storage.Message) *Metadata { return &Metadata{ Mailbox: m.Mailbox(), ID: m.ID(), diff --git a/pkg/message/message.go b/pkg/message/message.go index 3994ca3..7f8bec0 100644 --- a/pkg/message/message.go +++ b/pkg/message/message.go @@ -34,7 +34,7 @@ type Delivery struct { Reader io.Reader } -var _ storage.StoreMessage = &Delivery{} +var _ storage.Message = &Delivery{} // Mailbox getter. func (d *Delivery) Mailbox() string { @@ -71,7 +71,7 @@ func (d *Delivery) Size() int64 { return d.Meta.Size } -// RawReader contains the raw content of the message. -func (d *Delivery) RawReader() (io.ReadCloser, error) { +// Source contains the raw content of the message. +func (d *Delivery) Source() (io.ReadCloser, error) { return ioutil.NopCloser(d.Reader), nil } diff --git a/pkg/server/pop3/handler.go b/pkg/server/pop3/handler.go index c022619..f8229ca 100644 --- a/pkg/server/pop3/handler.go +++ b/pkg/server/pop3/handler.go @@ -57,17 +57,17 @@ var commands = map[string]bool{ // Session defines an active POP3 session type Session struct { - server *Server // Reference to the server we belong to - id int // Session ID number - conn net.Conn // Our network connection - remoteHost string // IP address of client - sendError error // Used to bail out of read loop on send error - state State // Current session state - reader *bufio.Reader // Buffered reader for our net conn - user string // Mailbox name - messages []storage.StoreMessage // Slice of messages in mailbox - retain []bool // Messages to retain upon UPDATE (true=retain) - msgCount int // Number of undeleted messages + server *Server // Reference to the server we belong to + id int // Session ID number + conn net.Conn // Our network connection + remoteHost string // IP address of client + sendError error // Used to bail out of read loop on send error + state State // Current session state + reader *bufio.Reader // Buffered reader for our net conn + user string // Mailbox name + messages []storage.Message // Slice of messages in mailbox + retain []bool // Messages to retain upon UPDATE (true=retain) + msgCount int // Number of undeleted messages } // NewSession creates a new POP3 session @@ -415,8 +415,8 @@ func (ses *Session) transactionHandler(cmd string, args []string) { } // Send the contents of the message to the client -func (ses *Session) sendMessage(msg storage.StoreMessage) { - reader, err := msg.RawReader() +func (ses *Session) sendMessage(msg storage.Message) { + reader, err := msg.Source() if err != nil { ses.logError("Failed to read message for RETR command") ses.send("-ERR Failed to RETR that message, internal error") @@ -448,8 +448,8 @@ func (ses *Session) sendMessage(msg storage.StoreMessage) { } // Send the headers plus the top N lines to the client -func (ses *Session) sendMessageTop(msg storage.StoreMessage, lineCount int) { - reader, err := msg.RawReader() +func (ses *Session) sendMessageTop(msg storage.Message, lineCount int) { + reader, err := msg.Source() if err != nil { ses.logError("Failed to read message for RETR command") ses.send("-ERR Failed to RETR that message, internal error") diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index cf06d5c..cf375b4 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -90,8 +90,8 @@ func (m *Message) rawPath() string { return filepath.Join(m.mailbox.path, m.Fid+".raw") } -// RawReader opens the .raw portion of a Message as an io.ReadCloser -func (m *Message) RawReader() (reader io.ReadCloser, err error) { +// Source opens the .raw portion of a Message as an io.ReadCloser +func (m *Message) Source() (reader io.ReadCloser, err error) { file, err := os.Open(m.rawPath()) if err != nil { return nil, err diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index f6a60ec..ebe5c7e 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -75,14 +75,14 @@ func New(cfg config.DataStoreConfig) storage.Store { } // AddMessage adds a message to the specified mailbox. -func (fs *Store) AddMessage(m storage.StoreMessage) (id string, err error) { +func (fs *Store) AddMessage(m storage.Message) (id string, err error) { mb, err := fs.mbox(m.Mailbox()) if err != nil { return "", err } mb.Lock() defer mb.Unlock() - r, err := m.RawReader() + r, err := m.Source() if err != nil { return "", err } @@ -136,7 +136,7 @@ func (fs *Store) AddMessage(m storage.StoreMessage) (id string, err error) { } // GetMessage returns the messages in the named mailbox, or an error. -func (fs *Store) GetMessage(mailbox, id string) (storage.StoreMessage, error) { +func (fs *Store) GetMessage(mailbox, id string) (storage.Message, error) { mb, err := fs.mbox(mailbox) if err != nil { return nil, err @@ -147,7 +147,7 @@ func (fs *Store) GetMessage(mailbox, id string) (storage.StoreMessage, error) { } // GetMessages returns the messages in the named mailbox, or an error. -func (fs *Store) GetMessages(mailbox string) ([]storage.StoreMessage, error) { +func (fs *Store) GetMessages(mailbox string) ([]storage.Message, error) { mb, err := fs.mbox(mailbox) if err != nil { return nil, err @@ -181,7 +181,7 @@ func (fs *Store) PurgeMessages(mailbox string) error { // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it // continues to return true. -func (fs *Store) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) error { +func (fs *Store) VisitMailboxes(f func([]storage.Message) (cont bool)) error { infos1, err := ioutil.ReadDir(fs.mailPath) if err != nil { return err diff --git a/pkg/storage/file/fstore_test.go b/pkg/storage/file/fstore_test.go index 18b1fe7..589bf18 100644 --- a/pkg/storage/file/fstore_test.go +++ b/pkg/storage/file/fstore_test.go @@ -133,7 +133,7 @@ func TestFSMissing(t *testing.T) { assert.Nil(t, err) // Try to read parts of message - _, err = msg.RawReader() + _, err = msg.Source() assert.Error(t, err) if t.Failed() { diff --git a/pkg/storage/file/mbox.go b/pkg/storage/file/mbox.go index ea6d7f4..cb5c1b9 100644 --- a/pkg/storage/file/mbox.go +++ b/pkg/storage/file/mbox.go @@ -28,13 +28,13 @@ type mbox struct { // getMessages scans the mailbox directory for .gob files and decodes them into // a slice of Message objects. -func (mb *mbox) getMessages() ([]storage.StoreMessage, error) { +func (mb *mbox) getMessages() ([]storage.Message, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err } } - messages := make([]storage.StoreMessage, len(mb.messages)) + messages := make([]storage.Message, len(mb.messages)) for i, m := range mb.messages { messages[i] = m } @@ -42,7 +42,7 @@ func (mb *mbox) getMessages() ([]storage.StoreMessage, error) { } // getMessage decodes a single message by ID and returns a Message object. -func (mb *mbox) getMessage(id string) (storage.StoreMessage, error) { +func (mb *mbox) getMessage(id string) (storage.Message, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err diff --git a/pkg/storage/retention.go b/pkg/storage/retention.go index 2da706e..6c7adb0 100644 --- a/pkg/storage/retention.go +++ b/pkg/storage/retention.go @@ -119,7 +119,7 @@ func (rs *RetentionScanner) DoScan() error { cutoff := time.Now().Add(-1 * rs.retentionPeriod) retained := 0 // Loop over all mailboxes. - err := rs.ds.VisitMailboxes(func(messages []StoreMessage) bool { + err := rs.ds.VisitMailboxes(func(messages []Message) bool { for _, msg := range messages { if msg.Date().Before(cutoff) { log.Tracef("Purging expired message %v/%v", msg.Mailbox(), msg.ID()) diff --git a/pkg/storage/retention_test.go b/pkg/storage/retention_test.go index a49cc59..234a377 100644 --- a/pkg/storage/retention_test.go +++ b/pkg/storage/retention_test.go @@ -37,13 +37,13 @@ func TestDoRetentionScan(t *testing.T) { t.Error(err) } // Delete should not have been called on new messages - for _, m := range []storage.StoreMessage{new1, new2, new3} { + for _, m := range []storage.Message{new1, new2, new3} { if ds.MessageDeleted(m) { t.Errorf("Expected %v to be present, was deleted", m.ID()) } } // Delete should have been called once on old messages - for _, m := range []storage.StoreMessage{old1, old2, old3} { + for _, m := range []storage.Message{old1, old2, old3} { if !ds.MessageDeleted(m) { t.Errorf("Expected %v to be deleted, was present", m.ID()) } @@ -51,7 +51,7 @@ func TestDoRetentionScan(t *testing.T) { } // stubMessage creates a message stub of a specific age -func stubMessage(mailbox string, ageHours int) storage.StoreMessage { +func stubMessage(mailbox string, ageHours int) storage.Message { return &message.Delivery{ Meta: message.Metadata{ Mailbox: mailbox, diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index f8bcaef..edc4afd 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -19,22 +19,22 @@ var ( // Store is the interface Inbucket uses to interact with storage implementations. type Store interface { // AddMessage stores the message, message ID and Size will be ignored. - AddMessage(message StoreMessage) (id string, err error) - GetMessage(mailbox, id string) (StoreMessage, error) - GetMessages(mailbox string) ([]StoreMessage, error) + AddMessage(message Message) (id string, err error) + GetMessage(mailbox, id string) (Message, error) + GetMessages(mailbox string) ([]Message, error) PurgeMessages(mailbox string) error RemoveMessage(mailbox, id string) error - VisitMailboxes(f func([]StoreMessage) (cont bool)) error + VisitMailboxes(f func([]Message) (cont bool)) error } -// StoreMessage represents a message to be stored, or returned from a storage implementation. -type StoreMessage interface { +// Message represents a message to be stored, or returned from a storage implementation. +type Message interface { Mailbox() string ID() string From() *mail.Address To() []*mail.Address Date() time.Time Subject() string - RawReader() (reader io.ReadCloser, err error) + Source() (io.ReadCloser, error) Size() int64 } diff --git a/pkg/test/storage.go b/pkg/test/storage.go index 2c45b10..b52445d 100644 --- a/pkg/test/storage.go +++ b/pkg/test/storage.go @@ -9,20 +9,20 @@ import ( // StoreStub stubs storage.Store for testing. type StoreStub struct { storage.Store - mailboxes map[string][]storage.StoreMessage - deleted map[storage.StoreMessage]struct{} + mailboxes map[string][]storage.Message + deleted map[storage.Message]struct{} } // NewStore creates a new StoreStub. func NewStore() *StoreStub { return &StoreStub{ - mailboxes: make(map[string][]storage.StoreMessage), - deleted: make(map[storage.StoreMessage]struct{}), + mailboxes: make(map[string][]storage.Message), + deleted: make(map[storage.Message]struct{}), } } // AddMessage adds a message to the specified mailbox. -func (s *StoreStub) AddMessage(m storage.StoreMessage) (id string, err error) { +func (s *StoreStub) AddMessage(m storage.Message) (id string, err error) { mb := m.Mailbox() msgs := s.mailboxes[mb] s.mailboxes[mb] = append(msgs, m) @@ -30,7 +30,7 @@ func (s *StoreStub) AddMessage(m storage.StoreMessage) (id string, err error) { } // GetMessage gets a message by ID from the specified mailbox. -func (s *StoreStub) GetMessage(mailbox, id string) (storage.StoreMessage, error) { +func (s *StoreStub) GetMessage(mailbox, id string) (storage.Message, error) { if mailbox == "messageerr" { return nil, errors.New("internal error") } @@ -43,7 +43,7 @@ func (s *StoreStub) GetMessage(mailbox, id string) (storage.StoreMessage, error) } // GetMessages gets all the messages for the specified mailbox. -func (s *StoreStub) GetMessages(mailbox string) ([]storage.StoreMessage, error) { +func (s *StoreStub) GetMessages(mailbox string) ([]storage.Message, error) { if mailbox == "messageserr" { return nil, errors.New("internal error") } @@ -54,7 +54,7 @@ func (s *StoreStub) GetMessages(mailbox string) ([]storage.StoreMessage, error) func (s *StoreStub) RemoveMessage(mailbox, id string) error { mb, ok := s.mailboxes[mailbox] if ok { - var msg storage.StoreMessage + var msg storage.Message for i, m := range mb { if m.ID() == id { msg = m @@ -72,7 +72,7 @@ func (s *StoreStub) RemoveMessage(mailbox, id string) error { // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it // continues to return true. -func (s *StoreStub) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) error { +func (s *StoreStub) VisitMailboxes(f func([]storage.Message) (cont bool)) error { for _, v := range s.mailboxes { if !f(v) { return nil @@ -82,7 +82,7 @@ func (s *StoreStub) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) e } // MessageDeleted returns true if the specified message was deleted -func (s *StoreStub) MessageDeleted(m storage.StoreMessage) bool { +func (s *StoreStub) MessageDeleted(m storage.Message) bool { _, ok := s.deleted[m] return ok } diff --git a/pkg/test/storage_suite.go b/pkg/test/storage_suite.go index 80e8626..3d167b5 100644 --- a/pkg/test/storage_suite.go +++ b/pkg/test/storage_suite.go @@ -138,7 +138,7 @@ func testContent(t *testing.T, store storage.Store) { if err != nil { t.Fatal(err) } - r, err := m.RawReader() + r, err := m.Source() if err != nil { t.Fatal(err) } @@ -269,7 +269,7 @@ func testVisitMailboxes(t *testing.T, ds storage.Store) { deliverMessage(t, ds, name, "New Message", time.Now()) } seen := 0 - err := ds.VisitMailboxes(func(messages []storage.StoreMessage) bool { + err := ds.VisitMailboxes(func(messages []storage.Message) bool { seen++ count := len(messages) if count != 2 { @@ -317,7 +317,7 @@ func deliverMessage( // getAndCountMessages is a test helper that expects to receive count messages or fails the test, it // also checks return error. -func getAndCountMessages(t *testing.T, s storage.Store, mailbox string, count int) []storage.StoreMessage { +func getAndCountMessages(t *testing.T, s storage.Store, mailbox string, count int) []storage.Message { t.Helper() msgs, err := s.GetMessages(mailbox) if err != nil { From 5cb07d5780f7e6ff94036a9e8b9788a6b000f309 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 18 Mar 2018 12:08:40 -0700 Subject: [PATCH 23/26] rest: Refactor JSON result value testing --- pkg/rest/apiv1_controller_test.go | 95 ++++-------- pkg/rest/testutils_test.go | 232 ++++++++++++------------------ 2 files changed, 116 insertions(+), 211 deletions(-) diff --git a/pkg/rest/apiv1_controller_test.go b/pkg/rest/apiv1_controller_test.go index b31cfcb..4906d21 100644 --- a/pkg/rest/apiv1_controller_test.go +++ b/pkg/rest/apiv1_controller_test.go @@ -6,7 +6,6 @@ import ( "net/mail" "net/textproto" "os" - "strings" "testing" "time" @@ -68,22 +67,6 @@ func TestRestMailboxList(t *testing.T) { } // Test JSON message headers - data1 := &InputMessageData{ - Mailbox: "good", - ID: "0001", - From: "", - To: []string{""}, - Subject: "subject 1", - Date: time.Date(2012, 2, 1, 10, 11, 12, 253, time.FixedZone("PST", -800)), - } - data2 := &InputMessageData{ - Mailbox: "good", - ID: "0002", - From: "", - To: []string{""}, - Subject: "subject 2", - Date: time.Date(2012, 7, 1, 10, 11, 12, 253, time.FixedZone("PDT", -700)), - } meta1 := message.Metadata{ Mailbox: "good", ID: "0001", @@ -114,25 +97,6 @@ func TestRestMailboxList(t *testing.T) { } // Check JSON - got := w.Body.String() - testStrings := []string{ - `{"mailbox":"good","id":"0001","from":"\u003cfrom1@host\u003e",` + - `"to":["\u003cto1@host\u003e"],"subject":"subject 1",` + - `"date":"2012-02-01T10:11:12.000000253-00:13","size":0}`, - `{"mailbox":"good","id":"0002","from":"\u003cfrom2@host\u003e",` + - `"to":["\u003cto1@host\u003e"],"subject":"subject 2",` + - `"date":"2012-07-01T10:11:12.000000253-00:11","size":0}`, - } - for _, ts := range testStrings { - t.Run(ts, func(t *testing.T) { - if !strings.Contains(got, ts) { - t.Errorf("got:\n%s\nwant to contain:\n%s", got, ts) - } - }) - } - - // Check JSON - // TODO transitional while refactoring dec := json.NewDecoder(w.Body) var result []interface{} if err := dec.Decode(&result); err != nil { @@ -141,18 +105,21 @@ func TestRestMailboxList(t *testing.T) { if len(result) != 2 { t.Fatalf("Expected 2 results, got %v", len(result)) } - if errors := data1.CompareToJSONHeaderMap(result[0]); len(errors) > 0 { - t.Logf("%v", result[0]) - for _, e := range errors { - t.Error(e) - } - } - if errors := data2.CompareToJSONHeaderMap(result[1]); len(errors) > 0 { - t.Logf("%v", result[1]) - for _, e := range errors { - t.Error(e) - } - } + + decodedStringEquals(t, result, "[0]/mailbox", "good") + decodedStringEquals(t, result, "[0]/id", "0001") + decodedStringEquals(t, result, "[0]/from", "") + decodedStringEquals(t, result, "[0]/to/[0]", "") + decodedStringEquals(t, result, "[0]/subject", "subject 1") + decodedStringEquals(t, result, "[0]/date", "2012-02-01T10:11:12.000000253-00:13") + decodedNumberEquals(t, result, "[0]/size", 0) + decodedStringEquals(t, result, "[1]/mailbox", "good") + decodedStringEquals(t, result, "[1]/id", "0002") + decodedStringEquals(t, result, "[1]/from", "") + decodedStringEquals(t, result, "[1]/to/[0]", "") + decodedStringEquals(t, result, "[1]/subject", "subject 2") + decodedStringEquals(t, result, "[1]/date", "2012-07-01T10:11:12.000000253-00:11") + decodedNumberEquals(t, result, "[1]/size", 0) if t.Failed() { // Wait for handler to finish logging @@ -225,19 +192,6 @@ func TestRestMessage(t *testing.T) { }, }, } - data1 := &InputMessageData{ - Mailbox: "good", - ID: "0001", - From: "", - Subject: "subject 1", - Date: time.Date(2012, 2, 1, 10, 11, 12, 253, time.FixedZone("PST", -800)), - Header: mail.Header{ - "To": []string{"fred@fish.com", "keyword@nsa.gov"}, - "From": []string{"noreply@inbucket.org"}, - }, - Text: "This is some text", - HTML: "This is some HTML", - } mm.AddMessage("good", msg1) // Check return code @@ -251,19 +205,24 @@ func TestRestMessage(t *testing.T) { } // Check JSON - // TODO transitional while refactoring dec := json.NewDecoder(w.Body) var result map[string]interface{} if err := dec.Decode(&result); err != nil { t.Errorf("Failed to decode JSON: %v", err) } - if errors := data1.CompareToJSONMessageMap(result); len(errors) > 0 { - t.Logf("%v", result) - for _, e := range errors { - t.Error(e) - } - } + decodedStringEquals(t, result, "mailbox", "good") + decodedStringEquals(t, result, "id", "0001") + decodedStringEquals(t, result, "from", "") + decodedStringEquals(t, result, "to/[0]", "") + decodedStringEquals(t, result, "subject", "subject 1") + decodedStringEquals(t, result, "date", "2012-02-01T10:11:12.000000253-00:13") + decodedNumberEquals(t, result, "size", 0) + decodedStringEquals(t, result, "body/text", "This is some text") + decodedStringEquals(t, result, "body/html", "This is some HTML") + decodedStringEquals(t, result, "header/To/[0]", "fred@fish.com") + decodedStringEquals(t, result, "header/To/[1]", "keyword@nsa.gov") + decodedStringEquals(t, result, "header/From/[0]", "noreply@inbucket.org") if t.Failed() { // Wait for handler to finish logging diff --git a/pkg/rest/testutils_test.go b/pkg/rest/testutils_test.go index 23996a5..587c3b7 100644 --- a/pkg/rest/testutils_test.go +++ b/pkg/rest/testutils_test.go @@ -2,12 +2,12 @@ package rest import ( "bytes" - "fmt" "log" "net/http" "net/http/httptest" - "net/mail" - "time" + "strconv" + "strings" + "testing" "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/message" @@ -15,146 +15,6 @@ import ( "github.com/jhillyerd/inbucket/pkg/server/web" ) -type InputMessageData struct { - Mailbox, ID, From, Subject string - To []string - Date time.Time - Size int - Header mail.Header - HTML, Text string -} - -// isJSONStringEqual is a utility function to return a nicely formatted message when -// comparing a string to a value received from a JSON map. -func isJSONStringEqual(key, expected string, received interface{}) (message string, ok bool) { - if value, ok := received.(string); ok { - if expected == value { - return "", true - } - return fmt.Sprintf("Expected value of key %v to be %q, got %q", key, expected, value), false - } - return fmt.Sprintf("Expected value of key %v to be a string, got %T", key, received), false -} - -// isJSONNumberEqual is a utility function to return a nicely formatted message when -// comparing an float64 to a value received from a JSON map. -func isJSONNumberEqual(key string, expected float64, received interface{}) (message string, ok bool) { - if value, ok := received.(float64); ok { - if expected == value { - return "", true - } - return fmt.Sprintf("Expected %v to be %v, got %v", key, expected, value), false - } - return fmt.Sprintf("Expected %v to be a string, got %T", key, received), false -} - -// CompareToJSONHeaderMap compares InputMessageData to a header map decoded from JSON, -// returning a list of things that did not match. -func (d *InputMessageData) CompareToJSONHeaderMap(json interface{}) (errors []string) { - if m, ok := json.(map[string]interface{}); ok { - if msg, ok := isJSONStringEqual(mailboxKey, d.Mailbox, m[mailboxKey]); !ok { - errors = append(errors, msg) - } - if msg, ok := isJSONStringEqual(idKey, d.ID, m[idKey]); !ok { - errors = append(errors, msg) - } - if msg, ok := isJSONStringEqual(fromKey, d.From, m[fromKey]); !ok { - errors = append(errors, msg) - } - for i, inputTo := range d.To { - if msg, ok := isJSONStringEqual(toKey, inputTo, m[toKey].([]interface{})[i]); !ok { - errors = append(errors, msg) - } - } - if msg, ok := isJSONStringEqual(subjectKey, d.Subject, m[subjectKey]); !ok { - errors = append(errors, msg) - } - exDate := d.Date.Format("2006-01-02T15:04:05.999999999-07:00") - if msg, ok := isJSONStringEqual(dateKey, exDate, m[dateKey]); !ok { - errors = append(errors, msg) - } - if msg, ok := isJSONNumberEqual(sizeKey, float64(d.Size), m[sizeKey]); !ok { - errors = append(errors, msg) - } - return errors - } - panic(fmt.Sprintf("Expected map[string]interface{} in json, got %T", json)) -} - -// CompareToJSONMessageMap compares InputMessageData to a message map decoded from JSON, -// returning a list of things that did not match. -func (d *InputMessageData) CompareToJSONMessageMap(json interface{}) (errors []string) { - // We need to check the same values as header first - errors = d.CompareToJSONHeaderMap(json) - - if m, ok := json.(map[string]interface{}); ok { - // Get nested body map - if m[bodyKey] != nil { - if body, ok := m[bodyKey].(map[string]interface{}); ok { - if msg, ok := isJSONStringEqual(textKey, d.Text, body[textKey]); !ok { - errors = append(errors, msg) - } - if msg, ok := isJSONStringEqual(htmlKey, d.HTML, body[htmlKey]); !ok { - errors = append(errors, msg) - } - } else { - panic(fmt.Sprintf("Expected map[string]interface{} in json key %q, got %T", - bodyKey, m[bodyKey])) - } - } else { - errors = append(errors, fmt.Sprintf("Expected body in JSON %q but it was nil", bodyKey)) - } - exDate := d.Date.Format("2006-01-02T15:04:05.999999999-07:00") - if msg, ok := isJSONStringEqual(dateKey, exDate, m[dateKey]); !ok { - errors = append(errors, msg) - } - if msg, ok := isJSONNumberEqual(sizeKey, float64(d.Size), m[sizeKey]); !ok { - errors = append(errors, msg) - } - - // Get nested header map - if m[headerKey] != nil { - if header, ok := m[headerKey].(map[string]interface{}); ok { - // Loop over input (expected) header names - for name, keyInputHeaders := range d.Header { - // Make sure expected header name exists in received JSON - if keyOutputVals, ok := header[name]; ok { - if keyOutputHeaders, ok := keyOutputVals.([]interface{}); ok { - // Loop over input (expected) header values - for _, inputHeader := range keyInputHeaders { - hasValue := false - // Look for expected value in received headers - for _, outputHeader := range keyOutputHeaders { - if inputHeader == outputHeader { - hasValue = true - break - } - } - if !hasValue { - errors = append(errors, fmt.Sprintf( - "JSON %v[%q] missing value %q", headerKey, name, inputHeader)) - } - } - } else { - // keyOutputValues was not a slice of interface{} - panic(fmt.Sprintf("Expected []interface{} in %v[%q], got %T", headerKey, - name, keyOutputVals)) - } - } else { - errors = append(errors, fmt.Sprintf("JSON %v missing key %q", headerKey, name)) - } - } - } - } else { - errors = append(errors, fmt.Sprintf("Expected header in JSON %q but it was nil", headerKey)) - } - } else { - panic(fmt.Sprintf("Expected map[string]interface{} in json, got %T", json)) - } - - return errors -} - func testRestGet(url string) (*httptest.ResponseRecorder, error) { req, err := http.NewRequest("GET", url, nil) req.Header.Add("Accept", "application/json") @@ -184,3 +44,89 @@ func setupWebServer(mm message.Manager) *bytes.Buffer { return buf } + +func decodedNumberEquals(t *testing.T, json interface{}, path string, want float64) { + t.Helper() + els := strings.Split(path, "/") + val, msg := getDecodedPath(json, els...) + if msg != "" { + t.Errorf("JSON result%s", msg) + return + } + if got, ok := val.(float64); ok { + if got == want { + return + } + } + t.Errorf("JSON result/%s == %v (%T), want: %v", path, val, val, want) +} + +func decodedStringEquals(t *testing.T, json interface{}, path string, want string) { + t.Helper() + els := strings.Split(path, "/") + val, msg := getDecodedPath(json, els...) + if msg != "" { + t.Errorf("JSON result%s", msg) + return + } + if got, ok := val.(string); ok { + if got == want { + return + } + } + t.Errorf("JSON result/%s == %v (%T), want: %v", path, val, val, want) +} + +// getDecodedPath recursively navigates the specified path, returing the requested element. If +// something goes wrong, the returned string will contain an explanation. +// +// Named path elements require the parent element to be a map[string]interface{}, numbers in square +// brackets require the parent element to be a []interface{}. +// +// getDecodedPath(o, "users", "[1]", "name") +// +// is equivalent to the JavaScript: +// +// o.users[1].name +// +func getDecodedPath(o interface{}, path ...string) (interface{}, string) { + if len(path) == 0 { + return o, "" + } + if o == nil { + return nil, " is nil" + } + key := path[0] + present := false + var val interface{} + if key[0] == '[' { + // Expecting slice. + index, err := strconv.Atoi(strings.Trim(key, "[]")) + if err != nil { + return nil, "/" + key + " is not a slice index" + } + oslice, ok := o.([]interface{}) + if !ok { + return nil, " is not a slice" + } + if index >= len(oslice) { + return nil, "/" + key + " is out of bounds" + } + val, present = oslice[index], true + } else { + // Expecting map. + omap, ok := o.(map[string]interface{}) + if !ok { + return nil, " is not a map" + } + val, present = omap[key] + } + if !present { + return nil, "/" + key + " is missing" + } + result, msg := getDecodedPath(val, path[1:]...) + if msg != "" { + return nil, "/" + key + msg + } + return result, "" +} From 0d0e07da70cd15182cc94e44f2fa6e97aad31cc5 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 18 Mar 2018 13:58:47 -0700 Subject: [PATCH 24/26] file: Remove index and dir mutexes HashLock makes these redundant. #77 --- pkg/storage/file/fmessage.go | 7 +------ pkg/storage/file/fstore.go | 10 ---------- pkg/storage/file/mbox.go | 11 +---------- 3 files changed, 2 insertions(+), 26 deletions(-) diff --git a/pkg/storage/file/fmessage.go b/pkg/storage/file/fmessage.go index cf375b4..79d1a65 100644 --- a/pkg/storage/file/fmessage.go +++ b/pkg/storage/file/fmessage.go @@ -1,7 +1,6 @@ package file import ( - "bufio" "io" "net/mail" "os" @@ -22,10 +21,6 @@ type Message struct { Fto []*mail.Address Fsubject string Fsize int64 - // These are for creating new messages only - writable bool - writerFile *os.File - writer *bufio.Writer } // newMessage creates a new FileMessage object and sets the Date and ID fields. @@ -48,7 +43,7 @@ func (mb *mbox) newMessage() (*Message, error) { } date := time.Now() id := generateID(date) - return &Message{mailbox: mb, Fid: id, Fdate: date, writable: true}, nil + return &Message{mailbox: mb, Fid: id, Fdate: date}, nil } // Mailbox returns the name of the mailbox this message resides in. diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index ebe5c7e..57a26bf 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "os" "path/filepath" - "sync" "time" "github.com/jhillyerd/inbucket/pkg/config" @@ -21,15 +20,6 @@ import ( const indexFileName = "index.gob" var ( - // indexMx is locked while reading/writing an index file - // - // NOTE: This is a bottleneck because it's a single lock even if we have a - // million index files - indexMx = new(sync.RWMutex) - - // dirMx is locked while creating/removing directories - dirMx = new(sync.Mutex) - // countChannel is filled with a sequential numbers (0000..9999), which are // used by generateID() to generate unique message IDs. It's global // because we only want one regardless of the number of DataStore objects diff --git a/pkg/storage/file/mbox.go b/pkg/storage/file/mbox.go index cb5c1b9..9a59984 100644 --- a/pkg/storage/file/mbox.go +++ b/pkg/storage/file/mbox.go @@ -101,9 +101,6 @@ func (mb *mbox) purge() error { func (mb *mbox) readIndex() error { // Clear message slice, open index mb.messages = mb.messages[:0] - // Lock for reading - indexMx.RLock() - defer indexMx.RUnlock() // Check if index exists if _, err := os.Stat(mb.indexPath); err != nil { // Does not exist, but that's not an error in our world @@ -146,8 +143,6 @@ func (mb *mbox) readIndex() error { // writeIndex overwrites the index on disk with the current mailbox data func (mb *mbox) writeIndex() error { // Lock for writing - indexMx.Lock() - defer indexMx.Unlock() if len(mb.messages) > 0 { // Ensure mailbox directory exists if err := mb.createDir(); err != nil { @@ -189,8 +184,6 @@ func (mb *mbox) writeIndex() error { // createDir checks for the presence of the path for this mailbox, creates it if needed func (mb *mbox) createDir() error { - dirMx.Lock() - defer dirMx.Unlock() if _, err := os.Stat(mb.path); err != nil { if err := os.MkdirAll(mb.path, 0770); err != nil { log.Errorf("Failed to create directory %v, %v", mb.path, err) @@ -202,8 +195,6 @@ func (mb *mbox) createDir() error { // removeDir removes the mailbox, plus empty higher level directories func (mb *mbox) removeDir() error { - dirMx.Lock() - defer dirMx.Unlock() // remove mailbox dir, including index file if err := os.RemoveAll(mb.path); err != nil { return err @@ -217,7 +208,7 @@ func (mb *mbox) removeDir() error { } // removeDirIfEmpty will remove the specified directory if it contains no files or directories. -// Caller should hold dirMx. Returns true if dir was removed. +// Returns true if dir was removed. func removeDirIfEmpty(path string) (removed bool) { f, err := os.Open(path) if err != nil { From 30f5c163e4d4687b36c56ca0cb9ecdade62f69c7 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 18 Mar 2018 14:30:56 -0700 Subject: [PATCH 25/26] log: Add locking to prevent race --- pkg/log/logging.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/log/logging.go b/pkg/log/logging.go index 7a73e24..7c06a1e 100644 --- a/pkg/log/logging.go +++ b/pkg/log/logging.go @@ -5,6 +5,7 @@ import ( golog "log" "os" "strings" + "sync" ) // Level is used to indicate the severity of a log entry @@ -30,12 +31,16 @@ var ( // logf is the file we send log output to, will be nil for stderr or stdout logf *os.File + + mu sync.RWMutex ) // Initialize logging. If logfile is equal to "stderr" or "stdout", then // we will log to that output stream. Otherwise the specificed file will // opened for writing, and all log data will be placed in it. func Initialize(logfile string) error { + mu.Lock() + defer mu.Unlock() if logfile != "stderr" { // stderr is the go logging default if logfile == "stdout" { @@ -55,6 +60,8 @@ func Initialize(logfile string) error { // SetLogLevel sets MaxLevel based on the provided string func SetLogLevel(level string) (ok bool) { + mu.Lock() + defer mu.Unlock() switch strings.ToUpper(level) { case "ERROR": MaxLevel = ERROR @@ -73,12 +80,16 @@ func SetLogLevel(level string) (ok bool) { // Errorf logs a message to the 'standard' Logger (always), accepts format strings func Errorf(msg string, args ...interface{}) { + mu.RLock() + defer mu.RUnlock() msg = "[ERROR] " + msg golog.Printf(msg, args...) } // Warnf logs a message to the 'standard' Logger if MaxLevel is >= WARN, accepts format strings func Warnf(msg string, args ...interface{}) { + mu.RLock() + defer mu.RUnlock() if MaxLevel >= WARN { msg = "[WARN ] " + msg golog.Printf(msg, args...) @@ -87,6 +98,8 @@ func Warnf(msg string, args ...interface{}) { // Infof logs a message to the 'standard' Logger if MaxLevel is >= INFO, accepts format strings func Infof(msg string, args ...interface{}) { + mu.RLock() + defer mu.RUnlock() if MaxLevel >= INFO { msg = "[INFO ] " + msg golog.Printf(msg, args...) @@ -95,6 +108,8 @@ func Infof(msg string, args ...interface{}) { // Tracef logs a message to the 'standard' Logger if MaxLevel is >= TRACE, accepts format strings func Tracef(msg string, args ...interface{}) { + mu.RLock() + defer mu.RUnlock() if MaxLevel >= TRACE { msg = "[TRACE] " + msg golog.Printf(msg, args...) @@ -105,6 +120,8 @@ func Tracef(msg string, args ...interface{}) { // log rotation system the opportunity to move the existing log file out of the // way and have Inbucket create a new one. func Rotate() { + mu.Lock() + defer mu.Unlock() // Rotate logs if configured if logf != nil { closeLogFile() @@ -117,6 +134,8 @@ func Rotate() { // Close the log file if we have one open func Close() { + mu.Lock() + defer mu.Unlock() if logf != nil { closeLogFile() } From e5785e81aa4cafc7aceb77eaf955b3e0a1c3a2c8 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 18 Mar 2018 15:14:48 -0700 Subject: [PATCH 26/26] Update CHANGELOG for refactor --- CHANGELOG.md | 6 ++++++ pkg/message/manager.go | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7874198..e674d8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [Unreleased] + +### Changed +- Massive refactor of back-end code. Inbucket should now be both easier and + more enjoyable to work on. + ## [v1.3.1] - 2018-03-10 ### Fixed diff --git a/pkg/message/manager.go b/pkg/message/manager.go index 2a8bb47..9ff4d66 100644 --- a/pkg/message/manager.go +++ b/pkg/message/manager.go @@ -45,7 +45,8 @@ func (s *StoreManager) Deliver( prefix string, source []byte, ) (string, error) { - // TODO enmime is too heavy for this step, only need header + // TODO enmime is too heavy for this step, only need header. + // Go's header parsing isn't good enough, so this is blocked on enmime issue #64. env, err := enmime.ReadEnvelope(bytes.NewReader(source)) if err != nil { return "", err