Reorganize packages, closes #79

- All packages go into either cmd or pkg directories
- Most packages renamed
- Server packages moved into pkg/server
- sanitize moved into webui, as that's the only place it's used
- filestore moved into pkg/storage/file
- Makefile updated, and PKG variable use fixed

pkg/storage/file/fmessage.go

@@ -0,0 +1,270 @@
+package filestore
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/mail"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/jhillyerd/enmime"
+	"github.com/jhillyerd/inbucket/pkg/storage"
+	"github.com/jhillyerd/inbucket/pkg/log"
+)
+
+// FileMessage implements Message and contains a little bit of data about a
+// particular email message, and methods to retrieve the rest of it from disk.
+type FileMessage struct {
+	mailbox *FileMailbox
+	// Stored in GOB
+	Fid      string
+	Fdate    time.Time
+	Ffrom    string
+	Fto      []string
+	Fsubject string
+	Fsize    int64
+	// These are for creating new messages only
+	writable   bool
+	writerFile *os.File
+	writer     *bufio.Writer
+}
+
+// NewMessage creates a new FileMessage object and sets the Date and Id fields.
+// It will also delete messages over messageCap if configured.
+func (mb *FileMailbox) NewMessage() (datastore.Message, error) {
+	// Load index
+	if !mb.indexLoaded {
+		if err := mb.readIndex(); err != nil {
+			return nil, err
+		}
+	}
+
+	// Delete old messages over messageCap
+	if mb.store.messageCap > 0 {
+		for len(mb.messages) >= mb.store.messageCap {
+			log.Infof("Mailbox %q over configured message cap", mb.name)
+			if err := mb.messages[0].Delete(); err != nil {
+				log.Errorf("Error deleting message: %s", err)
+			}
+		}
+	}
+
+	date := time.Now()
+	id := generateID(date)
+	return &FileMessage{mailbox: mb, Fid: id, Fdate: date, writable: true}, nil
+}
+
+// ID gets the ID of the Message
+func (m *FileMessage) ID() string {
+	return m.Fid
+}
+
+// Date returns the date/time this Message was received by Inbucket
+func (m *FileMessage) Date() time.Time {
+	return m.Fdate
+}
+
+// From returns the value of the Message From header
+func (m *FileMessage) From() string {
+	return m.Ffrom
+}
+
+// To returns the value of the Message To header
+func (m *FileMessage) To() []string {
+	return m.Fto
+}
+
+// Subject returns the value of the Message Subject header
+func (m *FileMessage) Subject() string {
+	return m.Fsubject
+}
+
+// String returns a string in the form: "Subject()" from From()
+func (m *FileMessage) String() string {
+	return fmt.Sprintf("\"%v\" from %v", m.Fsubject, m.Ffrom)
+}
+
+// Size returns the size of the Message on disk in bytes
+func (m *FileMessage) Size() int64 {
+	return m.Fsize
+}
+
+func (m *FileMessage) rawPath() string {
+	return filepath.Join(m.mailbox.path, m.Fid+".raw")
+}
+
+// ReadHeader opens the .raw portion of a Message and returns a standard Go mail.Message object
+func (m *FileMessage) ReadHeader() (msg *mail.Message, err error) {
+	file, err := os.Open(m.rawPath())
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err := file.Close(); err != nil {
+			log.Errorf("Failed to close %q: %v", m.rawPath(), err)
+		}
+	}()
+
+	reader := bufio.NewReader(file)
+	return mail.ReadMessage(reader)
+}
+
+// ReadBody opens the .raw portion of a Message and returns a MIMEBody object
+func (m *FileMessage) ReadBody() (body *enmime.Envelope, err error) {
+	file, err := os.Open(m.rawPath())
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err := file.Close(); err != nil {
+			log.Errorf("Failed to close %q: %v", m.rawPath(), err)
+		}
+	}()
+
+	reader := bufio.NewReader(file)
+	mime, err := enmime.ReadEnvelope(reader)
+	if err != nil {
+		return nil, err
+	}
+	return mime, nil
+}
+
+// RawReader opens the .raw portion of a Message as an io.ReadCloser
+func (m *FileMessage) RawReader() (reader io.ReadCloser, err error) {
+	file, err := os.Open(m.rawPath())
+	if err != nil {
+		return nil, err
+	}
+	return file, nil
+}
+
+// ReadRaw opens the .raw portion of a Message and returns it as a string
+func (m *FileMessage) ReadRaw() (raw *string, err error) {
+	reader, err := m.RawReader()
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err := reader.Close(); err != nil {
+			log.Errorf("Failed to close %q: %v", m.rawPath(), err)
+		}
+	}()
+
+	bodyBytes, err := ioutil.ReadAll(bufio.NewReader(reader))
+	if err != nil {
+		return nil, err
+	}
+	bodyString := string(bodyBytes)
+	return &bodyString, nil
+}
+
+// Append data to a newly opened Message, this will fail on a pre-existing Message and
+// after Close() is called.
+func (m *FileMessage) Append(data []byte) error {
+	// Prevent Appending to a pre-existing Message
+	if !m.writable {
+		return datastore.ErrNotWritable
+	}
+	// Open file for writing if we haven't yet
+	if m.writer == nil {
+		// Ensure mailbox directory exists
+		if err := m.mailbox.createDir(); err != nil {
+			return err
+		}
+		file, err := os.Create(m.rawPath())
+		if err != nil {
+			// Set writable false just in case something calls me a million times
+			m.writable = false
+			return err
+		}
+		m.writerFile = file
+		m.writer = bufio.NewWriter(file)
+	}
+	_, err := m.writer.Write(data)
+	m.Fsize += int64(len(data))
+	return err
+}
+
+// Close this Message for writing - no more data may be Appended.  Close() will also
+// trigger the creation of the .gob file.
+func (m *FileMessage) Close() error {
+	// nil out the writer fields so they can't be used
+	writer := m.writer
+	writerFile := m.writerFile
+	m.writer = nil
+	m.writerFile = nil
+
+	if writer != nil {
+		if err := writer.Flush(); err != nil {
+			return err
+		}
+	}
+	if writerFile != nil {
+		if err := writerFile.Close(); err != nil {
+			return err
+		}
+	}
+
+	// Fetch headers
+	body, err := m.ReadBody()
+	if err != nil {
+		return err
+	}
+
+	// Only public fields are stored in gob, hence starting with capital F
+	// Parse From address
+	if address, err := mail.ParseAddress(body.GetHeader("From")); err == nil {
+		m.Ffrom = address.String()
+	} else {
+		m.Ffrom = body.GetHeader("From")
+	}
+	m.Fsubject = body.GetHeader("Subject")
+
+	// Turn the To header into a slice
+	if addresses, err := body.AddressList("To"); err == nil {
+		for _, a := range addresses {
+			m.Fto = append(m.Fto, a.String())
+		}
+	} else {
+		m.Fto = []string{body.GetHeader("To")}
+	}
+
+	// Refresh the index before adding our message
+	err = m.mailbox.readIndex()
+	if err != nil {
+		return err
+	}
+
+	// Made it this far without errors, add it to the index
+	m.mailbox.messages = append(m.mailbox.messages, m)
+	return m.mailbox.writeIndex()
+}
+
+// Delete this Message from disk by removing it from the index and deleting the
+// raw files.
+func (m *FileMessage) Delete() error {
+	messages := m.mailbox.messages
+	for i, mm := range messages {
+		if m == mm {
+			// Slice around message we are deleting
+			m.mailbox.messages = append(messages[:i], messages[i+1:]...)
+			break
+		}
+	}
+	if err := m.mailbox.writeIndex(); err != nil {
+		return err
+	}
+
+	if len(m.mailbox.messages) == 0 {
+		// This was the last message, thus writeIndex() has removed the entire
+		// directory; we don't need to delete the raw file.
+		return nil
+	}
+
+	// There are still messages in the index
+	log.Tracef("Deleting %v", m.rawPath())
+	return os.Remove(m.rawPath())
+}

pkg/storage/file/fstore.go

@@ -0,0 +1,367 @@
+package filestore
+
+import (
+	"bufio"
+	"encoding/gob"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/jhillyerd/inbucket/pkg/config"
+	"github.com/jhillyerd/inbucket/pkg/storage"
+	"github.com/jhillyerd/inbucket/pkg/log"
+	"github.com/jhillyerd/inbucket/pkg/stringutil"
+)
+
+// Name of index file in each mailbox
+const indexFileName = "index.gob"
+
+var (
+	// indexMx is locked while reading/writing an index file
+	//
+	// NOTE: This is a bottleneck because it's a single lock even if we have a
+	// million index files
+	indexMx = new(sync.RWMutex)
+
+	// dirMx is locked while creating/removing directories
+	dirMx = new(sync.Mutex)
+
+	// countChannel is filled with a sequential numbers (0000..9999), which are
+	// used by generateID() to generate unique message IDs.  It's global
+	// because we only want one regardless of the number of DataStore objects
+	countChannel = make(chan int, 10)
+)
+
+func init() {
+	// Start generator
+	go countGenerator(countChannel)
+}
+
+// Populates the channel with numbers
+func countGenerator(c chan int) {
+	for i := 0; true; i = (i + 1) % 10000 {
+		c <- i
+	}
+}
+
+// FileDataStore implements DataStore aand is the root of the mail storage
+// hiearchy.  It provides access to Mailbox objects
+type FileDataStore struct {
+	hashLock   datastore.HashLock
+	path       string
+	mailPath   string
+	messageCap int
+}
+
+// NewFileDataStore creates a new DataStore object using the specified path
+func NewFileDataStore(cfg config.DataStoreConfig) datastore.DataStore {
+	path := cfg.Path
+	if path == "" {
+		log.Errorf("No value configured for datastore path")
+		return nil
+	}
+	mailPath := filepath.Join(path, "mail")
+	if _, err := os.Stat(mailPath); err != nil {
+		// Mail datastore does not yet exist
+		if err = os.MkdirAll(mailPath, 0770); err != nil {
+			log.Errorf("Error creating dir %q: %v", mailPath, err)
+		}
+	}
+	return &FileDataStore{path: path, mailPath: mailPath, messageCap: cfg.MailboxMsgCap}
+}
+
+// DefaultFileDataStore creates a new DataStore object.  It uses the inbucket.Config object to
+// construct it's path.
+func DefaultFileDataStore() datastore.DataStore {
+	cfg := config.GetDataStoreConfig()
+	return NewFileDataStore(cfg)
+}
+
+// MailboxFor retrieves the Mailbox object for a specified email address, if the mailbox
+// does not exist, it will attempt to create it.
+func (ds *FileDataStore) MailboxFor(emailAddress string) (datastore.Mailbox, error) {
+	name, err := stringutil.ParseMailboxName(emailAddress)
+	if err != nil {
+		return nil, err
+	}
+	dir := stringutil.HashMailboxName(name)
+	s1 := dir[0:3]
+	s2 := dir[0:6]
+	path := filepath.Join(ds.mailPath, s1, s2, dir)
+	indexPath := filepath.Join(path, indexFileName)
+
+	return &FileMailbox{store: ds, name: name, dirName: dir, path: path,
+		indexPath: indexPath}, nil
+}
+
+// AllMailboxes returns a slice with all Mailboxes
+func (ds *FileDataStore) AllMailboxes() ([]datastore.Mailbox, error) {
+	mailboxes := make([]datastore.Mailbox, 0, 100)
+	infos1, err := ioutil.ReadDir(ds.mailPath)
+	if err != nil {
+		return nil, err
+	}
+	// Loop over level 1 directories
+	for _, inf1 := range infos1 {
+		if inf1.IsDir() {
+			l1 := inf1.Name()
+			infos2, err := ioutil.ReadDir(filepath.Join(ds.mailPath, l1))
+			if err != nil {
+				return nil, err
+			}
+			// Loop over level 2 directories
+			for _, inf2 := range infos2 {
+				if inf2.IsDir() {
+					l2 := inf2.Name()
+					infos3, err := ioutil.ReadDir(filepath.Join(ds.mailPath, l1, l2))
+					if err != nil {
+						return nil, err
+					}
+					// Loop over mailboxes
+					for _, inf3 := range infos3 {
+						if inf3.IsDir() {
+							mbdir := inf3.Name()
+							mbpath := filepath.Join(ds.mailPath, l1, l2, mbdir)
+							idx := filepath.Join(mbpath, indexFileName)
+							mb := &FileMailbox{store: ds, dirName: mbdir, path: mbpath,
+								indexPath: idx}
+							mailboxes = append(mailboxes, mb)
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return mailboxes, nil
+}
+
+func (ds *FileDataStore) LockFor(emailAddress string) (*sync.RWMutex, error) {
+	name, err := stringutil.ParseMailboxName(emailAddress)
+	if err != nil {
+		return nil, err
+	}
+	hash := stringutil.HashMailboxName(name)
+	return ds.hashLock.Get(hash), nil
+}
+
+// FileMailbox implements Mailbox, manages the mail for a specific user and
+// correlates to a particular directory on disk.
+type FileMailbox struct {
+	store       *FileDataStore
+	name        string
+	dirName     string
+	path        string
+	indexLoaded bool
+	indexPath   string
+	messages    []*FileMessage
+}
+
+// Name of the mailbox
+func (mb *FileMailbox) Name() string {
+	return mb.name
+}
+
+// String renders the name and directory path of the mailbox
+func (mb *FileMailbox) String() string {
+	return mb.name + "[" + mb.dirName + "]"
+}
+
+// GetMessages scans the mailbox directory for .gob files and decodes them into
+// a slice of Message objects.
+func (mb *FileMailbox) GetMessages() ([]datastore.Message, error) {
+	if !mb.indexLoaded {
+		if err := mb.readIndex(); err != nil {
+			return nil, err
+		}
+	}
+
+	messages := make([]datastore.Message, len(mb.messages))
+	for i, m := range mb.messages {
+		messages[i] = m
+	}
+	return messages, nil
+}
+
+// GetMessage decodes a single message by Id and returns a Message object
+func (mb *FileMailbox) GetMessage(id string) (datastore.Message, error) {
+	if !mb.indexLoaded {
+		if err := mb.readIndex(); err != nil {
+			return nil, err
+		}
+	}
+
+	if id == "latest" && len(mb.messages) != 0 {
+		return mb.messages[len(mb.messages)-1], nil
+	}
+
+	for _, m := range mb.messages {
+		if m.Fid == id {
+			return m, nil
+		}
+	}
+
+	return nil, datastore.ErrNotExist
+}
+
+// Purge deletes all messages in this mailbox
+func (mb *FileMailbox) Purge() error {
+	mb.messages = mb.messages[:0]
+	return mb.writeIndex()
+}
+
+// readIndex loads the mailbox index data from disk
+func (mb *FileMailbox) readIndex() error {
+	// Clear message slice, open index
+	mb.messages = mb.messages[:0]
+	// Lock for reading
+	indexMx.RLock()
+	defer indexMx.RUnlock()
+	// Check if index exists
+	if _, err := os.Stat(mb.indexPath); err != nil {
+		// Does not exist, but that's not an error in our world
+		log.Tracef("Index %v does not exist (yet)", mb.indexPath)
+		mb.indexLoaded = true
+		return nil
+	}
+	file, err := os.Open(mb.indexPath)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := file.Close(); err != nil {
+			log.Errorf("Failed to close %q: %v", mb.indexPath, err)
+		}
+	}()
+
+	// Decode gob data
+	dec := gob.NewDecoder(bufio.NewReader(file))
+	for {
+		msg := new(FileMessage)
+		if err = dec.Decode(msg); err != nil {
+			if err == io.EOF {
+				// It's OK to get an EOF here
+				break
+			}
+			return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err)
+		}
+		msg.mailbox = mb
+		mb.messages = append(mb.messages, msg)
+	}
+
+	mb.indexLoaded = true
+	return nil
+}
+
+// writeIndex overwrites the index on disk with the current mailbox data
+func (mb *FileMailbox) writeIndex() error {
+	// Lock for writing
+	indexMx.Lock()
+	defer indexMx.Unlock()
+	if len(mb.messages) > 0 {
+		// Ensure mailbox directory exists
+		if err := mb.createDir(); err != nil {
+			return err
+		}
+		// Open index for writing
+		file, err := os.Create(mb.indexPath)
+		if err != nil {
+			return err
+		}
+		writer := bufio.NewWriter(file)
+		// Write each message and then flush
+		enc := gob.NewEncoder(writer)
+		for _, m := range mb.messages {
+			err = enc.Encode(m)
+			if err != nil {
+				_ = file.Close()
+				return err
+			}
+		}
+		if err := writer.Flush(); err != nil {
+			_ = file.Close()
+			return err
+		}
+		if err := file.Close(); err != nil {
+			log.Errorf("Failed to close %q: %v", mb.indexPath, err)
+			return err
+		}
+	} else {
+		// No messages, delete index+maildir
+		log.Tracef("Removing mailbox %v", mb.path)
+		return mb.removeDir()
+	}
+
+	return nil
+}
+
+// createDir checks for the presence of the path for this mailbox, creates it if needed
+func (mb *FileMailbox) createDir() error {
+	dirMx.Lock()
+	defer dirMx.Unlock()
+	if _, err := os.Stat(mb.path); err != nil {
+		if err := os.MkdirAll(mb.path, 0770); err != nil {
+			log.Errorf("Failed to create directory %v, %v", mb.path, err)
+			return err
+		}
+	}
+	return nil
+}
+
+// removeDir removes the mailbox, plus empty higher level directories
+func (mb *FileMailbox) removeDir() error {
+	dirMx.Lock()
+	defer dirMx.Unlock()
+	// remove mailbox dir, including index file
+	if err := os.RemoveAll(mb.path); err != nil {
+		return err
+	}
+	// remove parents if empty
+	dir := filepath.Dir(mb.path)
+	if removeDirIfEmpty(dir) {
+		removeDirIfEmpty(filepath.Dir(dir))
+	}
+	return nil
+}
+
+// removeDirIfEmpty will remove the specified directory if it contains no files or directories.
+// Caller should hold dirMx.  Returns true if dir was removed.
+func removeDirIfEmpty(path string) (removed bool) {
+	f, err := os.Open(path)
+	if err != nil {
+		return false
+	}
+	files, err := f.Readdirnames(0)
+	_ = f.Close()
+	if err != nil {
+		return false
+	}
+	if len(files) > 0 {
+		// Dir not empty
+		return false
+	}
+	log.Tracef("Removing dir %v", path)
+	err = os.Remove(path)
+	if err != nil {
+		log.Errorf("Failed to remove %q: %v", path, err)
+		return false
+	}
+	return true
+}
+
+// generatePrefix converts a Time object into the ISO style format we use
+// as a prefix for message files.  Note:  It is used directly by unit
+// tests.
+func generatePrefix(date time.Time) string {
+	return date.Format("20060102T150405")
+}
+
+// generateId adds a 4-digit unique number onto the end of the string
+// returned by generatePrefix()
+func generateID(date time.Time) string {
+	return generatePrefix(date) + "-" + fmt.Sprintf("%04d", <-countChannel)
+}

pkg/storage/file/fstore_test.go

@@ -0,0 +1,583 @@
+package filestore
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/jhillyerd/inbucket/pkg/config"
+	"github.com/stretchr/testify/assert"
+)
+
+// Test directory structure created by filestore
+func TestFSDirStructure(t *testing.T) {
+	ds, logbuf := setupDataStore(config.DataStoreConfig{})
+	defer teardownDataStore(ds)
+	root := ds.path
+
+	// james hashes to 474ba67bdb289c6263b36dfd8a7bed6c85b04943
+	mbName := "james"
+
+	// Check filestore root exists
+	assert.True(t, isDir(root), "Expected %q to be a directory", root)
+
+	// Check mail dir exists
+	expect := filepath.Join(root, "mail")
+	assert.True(t, isDir(expect), "Expected %q to be a directory", expect)
+
+	// Check first hash section does not exist
+	expect = filepath.Join(root, "mail", "474")
+	assert.False(t, isDir(expect), "Expected %q to not exist", expect)
+
+	// Deliver test message
+	id1, _ := deliverMessage(ds, mbName, "test", time.Now())
+
+	// Check path to message exists
+	assert.True(t, isDir(expect), "Expected %q to be a directory", expect)
+	expect = filepath.Join(expect, "474ba6")
+	assert.True(t, isDir(expect), "Expected %q to be a directory", expect)
+	expect = filepath.Join(expect, "474ba67bdb289c6263b36dfd8a7bed6c85b04943")
+	assert.True(t, isDir(expect), "Expected %q to be a directory", expect)
+
+	// Check files
+	mbPath := expect
+	expect = filepath.Join(mbPath, "index.gob")
+	assert.True(t, isFile(expect), "Expected %q to be a file", expect)
+	expect = filepath.Join(mbPath, id1+".raw")
+	assert.True(t, isFile(expect), "Expected %q to be a file", expect)
+
+	// Deliver second test message
+	id2, _ := deliverMessage(ds, mbName, "test 2", time.Now())
+
+	// Check files
+	expect = filepath.Join(mbPath, "index.gob")
+	assert.True(t, isFile(expect), "Expected %q to be a file", expect)
+	expect = filepath.Join(mbPath, id2+".raw")
+	assert.True(t, isFile(expect), "Expected %q to be a file", expect)
+
+	// Delete message
+	mb, err := ds.MailboxFor(mbName)
+	assert.Nil(t, err)
+	msg, err := mb.GetMessage(id1)
+	assert.Nil(t, err)
+	err = msg.Delete()
+	assert.Nil(t, err)
+
+	// Message should be removed
+	expect = filepath.Join(mbPath, id1+".raw")
+	assert.False(t, isPresent(expect), "Did not expect %q to exist", expect)
+	expect = filepath.Join(mbPath, "index.gob")
+	assert.True(t, isFile(expect), "Expected %q to be a file", expect)
+
+	// Delete message
+	msg, err = mb.GetMessage(id2)
+	assert.Nil(t, err)
+	err = msg.Delete()
+	assert.Nil(t, err)
+
+	// Message should be removed
+	expect = filepath.Join(mbPath, id2+".raw")
+	assert.False(t, isPresent(expect), "Did not expect %q to exist", expect)
+
+	// No messages, index & maildir should be removed
+	expect = filepath.Join(mbPath, "index.gob")
+	assert.False(t, isPresent(expect), "Did not expect %q to exist", expect)
+	expect = mbPath
+	assert.False(t, isPresent(expect), "Did not expect %q to exist", expect)
+
+	if t.Failed() {
+		// Wait for handler to finish logging
+		time.Sleep(2 * time.Second)
+		// Dump buffered log data if there was a failure
+		_, _ = io.Copy(os.Stderr, logbuf)
+	}
+}
+
+// Test FileDataStore.AllMailboxes()
+func TestFSAllMailboxes(t *testing.T) {
+	ds, logbuf := setupDataStore(config.DataStoreConfig{})
+	defer teardownDataStore(ds)
+
+	for _, name := range []string{"abby", "bill", "christa", "donald", "evelyn"} {
+		// Create day old message
+		date := time.Now().Add(-24 * time.Hour)
+		deliverMessage(ds, name, "Old Message", date)
+
+		// Create current message
+		date = time.Now()
+		deliverMessage(ds, name, "New Message", date)
+	}
+
+	mboxes, err := ds.AllMailboxes()
+	assert.Nil(t, err)
+	assert.Equal(t, len(mboxes), 5)
+
+	if t.Failed() {
+		// Wait for handler to finish logging
+		time.Sleep(2 * time.Second)
+		// Dump buffered log data if there was a failure
+		_, _ = io.Copy(os.Stderr, logbuf)
+	}
+}
+
+// Test delivering several messages to the same mailbox, meanwhile querying its
+// contents with a new mailbox object each time
+func TestFSDeliverMany(t *testing.T) {
+	ds, logbuf := setupDataStore(config.DataStoreConfig{})
+	defer teardownDataStore(ds)
+
+	mbName := "fred"
+	subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"}
+
+	for i, subj := range subjects {
+		// Check number of messages
+		mb, err := ds.MailboxFor(mbName)
+		if err != nil {
+			t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err)
+		}
+		msgs, err := mb.GetMessages()
+		if err != nil {
+			t.Fatalf("Failed to GetMessages for %q: %v", mbName, err)
+		}
+		assert.Equal(t, i, len(msgs), "Expected %v message(s), but got %v", i, len(msgs))
+
+		// Add a message
+		deliverMessage(ds, mbName, subj, time.Now())
+	}
+
+	mb, err := ds.MailboxFor(mbName)
+	if err != nil {
+		t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err)
+	}
+	msgs, err := mb.GetMessages()
+	if err != nil {
+		t.Fatalf("Failed to GetMessages for %q: %v", mbName, err)
+	}
+	assert.Equal(t, len(subjects), len(msgs), "Expected %v message(s), but got %v",
+		len(subjects), len(msgs))
+
+	// Confirm delivery order
+	for i, expect := range subjects {
+		subj := msgs[i].Subject()
+		assert.Equal(t, expect, subj, "Expected subject %q, got %q", expect, subj)
+	}
+
+	if t.Failed() {
+		// Wait for handler to finish logging
+		time.Sleep(2 * time.Second)
+		// Dump buffered log data if there was a failure
+		_, _ = io.Copy(os.Stderr, logbuf)
+	}
+}
+
+// Test deleting messages
+func TestFSDelete(t *testing.T) {
+	ds, logbuf := setupDataStore(config.DataStoreConfig{})
+	defer teardownDataStore(ds)
+
+	mbName := "fred"
+	subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"}
+
+	for _, subj := range subjects {
+		// Add a message
+		deliverMessage(ds, mbName, subj, time.Now())
+	}
+
+	mb, err := ds.MailboxFor(mbName)
+	if err != nil {
+		t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err)
+	}
+	msgs, err := mb.GetMessages()
+	if err != nil {
+		t.Fatalf("Failed to GetMessages for %q: %v", mbName, err)
+	}
+	assert.Equal(t, len(subjects), len(msgs), "Expected %v message(s), but got %v",
+		len(subjects), len(msgs))
+
+	// Delete a couple messages
+	_ = msgs[1].Delete()
+	_ = msgs[3].Delete()
+
+	// Confirm deletion
+	mb, err = ds.MailboxFor(mbName)
+	if err != nil {
+		t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err)
+	}
+	msgs, err = mb.GetMessages()
+	if err != nil {
+		t.Fatalf("Failed to GetMessages for %q: %v", mbName, err)
+	}
+
+	subjects = []string{"alpha", "charlie", "echo"}
+	assert.Equal(t, len(subjects), len(msgs), "Expected %v message(s), but got %v",
+		len(subjects), len(msgs))
+	for i, expect := range subjects {
+		subj := msgs[i].Subject()
+		assert.Equal(t, expect, subj, "Expected subject %q, got %q", expect, subj)
+	}
+
+	// Try appending one more
+	deliverMessage(ds, mbName, "foxtrot", time.Now())
+
+	mb, err = ds.MailboxFor(mbName)
+	if err != nil {
+		t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err)
+	}
+	msgs, err = mb.GetMessages()
+	if err != nil {
+		t.Fatalf("Failed to GetMessages for %q: %v", mbName, err)
+	}
+
+	subjects = []string{"alpha", "charlie", "echo", "foxtrot"}
+	assert.Equal(t, len(subjects), len(msgs), "Expected %v message(s), but got %v",
+		len(subjects), len(msgs))
+	for i, expect := range subjects {
+		subj := msgs[i].Subject()
+		assert.Equal(t, expect, subj, "Expected subject %q, got %q", expect, subj)
+	}
+
+	if t.Failed() {
+		// Wait for handler to finish logging
+		time.Sleep(2 * time.Second)
+		// Dump buffered log data if there was a failure
+		_, _ = io.Copy(os.Stderr, logbuf)
+	}
+}
+
+// Test purging a mailbox
+func TestFSPurge(t *testing.T) {
+	ds, logbuf := setupDataStore(config.DataStoreConfig{})
+	defer teardownDataStore(ds)
+
+	mbName := "fred"
+	subjects := []string{"alpha", "bravo", "charlie", "delta", "echo"}
+
+	for _, subj := range subjects {
+		// Add a message
+		deliverMessage(ds, mbName, subj, time.Now())
+	}
+
+	mb, err := ds.MailboxFor(mbName)
+	if err != nil {
+		t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err)
+	}
+	msgs, err := mb.GetMessages()
+	if err != nil {
+		t.Fatalf("Failed to GetMessages for %q: %v", mbName, err)
+	}
+	assert.Equal(t, len(subjects), len(msgs), "Expected %v message(s), but got %v",
+		len(subjects), len(msgs))
+
+	// Purge mailbox
+	err = mb.Purge()
+	assert.Nil(t, err)
+
+	// Confirm deletion
+	mb, err = ds.MailboxFor(mbName)
+	if err != nil {
+		t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err)
+	}
+	msgs, err = mb.GetMessages()
+	if err != nil {
+		t.Fatalf("Failed to GetMessages for %q: %v", mbName, err)
+	}
+
+	assert.Equal(t, len(msgs), 0, "Expected mailbox to have zero messages, got %v", len(msgs))
+
+	if t.Failed() {
+		// Wait for handler to finish logging
+		time.Sleep(2 * time.Second)
+		// Dump buffered log data if there was a failure
+		_, _ = io.Copy(os.Stderr, logbuf)
+	}
+}
+
+// Test message size calculation
+func TestFSSize(t *testing.T) {
+	ds, logbuf := setupDataStore(config.DataStoreConfig{})
+	defer teardownDataStore(ds)
+
+	mbName := "fred"
+	subjects := []string{"a", "br", "much longer than the others"}
+	sentIds := make([]string, len(subjects))
+	sentSizes := make([]int64, len(subjects))
+
+	for i, subj := range subjects {
+		// Add a message
+		id, size := deliverMessage(ds, mbName, subj, time.Now())
+		sentIds[i] = id
+		sentSizes[i] = size
+	}
+
+	mb, err := ds.MailboxFor(mbName)
+	if err != nil {
+		t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err)
+	}
+	for i, id := range sentIds {
+		msg, err := mb.GetMessage(id)
+		assert.Nil(t, err)
+
+		expect := sentSizes[i]
+		size := msg.Size()
+		assert.Equal(t, expect, size, "Expected size of %v, got %v", expect, size)
+	}
+
+	if t.Failed() {
+		// Wait for handler to finish logging
+		time.Sleep(2 * time.Second)
+		// Dump buffered log data if there was a failure
+		_, _ = io.Copy(os.Stderr, logbuf)
+	}
+}
+
+// Test missing files
+func TestFSMissing(t *testing.T) {
+	ds, logbuf := setupDataStore(config.DataStoreConfig{})
+	defer teardownDataStore(ds)
+
+	mbName := "fred"
+	subjects := []string{"a", "b", "c"}
+	sentIds := make([]string, len(subjects))
+
+	for i, subj := range subjects {
+		// Add a message
+		id, _ := deliverMessage(ds, mbName, subj, time.Now())
+		sentIds[i] = id
+	}
+
+	mb, err := ds.MailboxFor(mbName)
+	if err != nil {
+		t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err)
+	}
+
+	// Delete a message file without removing it from index
+	msg, err := mb.GetMessage(sentIds[1])
+	assert.Nil(t, err)
+	fmsg := msg.(*FileMessage)
+	_ = os.Remove(fmsg.rawPath())
+	msg, err = mb.GetMessage(sentIds[1])
+	assert.Nil(t, err)
+
+	// Try to read parts of message
+	_, err = msg.ReadHeader()
+	assert.Error(t, err)
+	_, err = msg.ReadBody()
+	assert.Error(t, err)
+
+	if t.Failed() {
+		// Wait for handler to finish logging
+		time.Sleep(2 * time.Second)
+		// Dump buffered log data if there was a failure
+		_, _ = io.Copy(os.Stderr, logbuf)
+	}
+}
+
+// Test delivering several messages to the same mailbox, see if message cap works
+func TestFSMessageCap(t *testing.T) {
+	mbCap := 10
+	ds, logbuf := setupDataStore(config.DataStoreConfig{MailboxMsgCap: mbCap})
+	defer teardownDataStore(ds)
+
+	mbName := "captain"
+	for i := 0; i < 20; i++ {
+		// Add a message
+		subj := fmt.Sprintf("subject %v", i)
+		deliverMessage(ds, mbName, subj, time.Now())
+		t.Logf("Delivered %q", subj)
+
+		// Check number of messages
+		mb, err := ds.MailboxFor(mbName)
+		if err != nil {
+			t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err)
+		}
+		msgs, err := mb.GetMessages()
+		if err != nil {
+			t.Fatalf("Failed to GetMessages for %q: %v", mbName, err)
+		}
+		if len(msgs) > mbCap {
+			t.Errorf("Mailbox should be capped at %v messages, but has %v", mbCap, len(msgs))
+		}
+
+		// Check that the first message is correct
+		first := i - mbCap + 1
+		if first < 0 {
+			first = 0
+		}
+		firstSubj := fmt.Sprintf("subject %v", first)
+		if firstSubj != msgs[0].Subject() {
+			t.Errorf("Expected first subject to be %q, got %q", firstSubj, msgs[0].Subject())
+		}
+	}
+
+	if t.Failed() {
+		// Wait for handler to finish logging
+		time.Sleep(2 * time.Second)
+		// Dump buffered log data if there was a failure
+		_, _ = io.Copy(os.Stderr, logbuf)
+	}
+}
+
+// Test delivering several messages to the same mailbox, see if no message cap works
+func TestFSNoMessageCap(t *testing.T) {
+	mbCap := 0
+	ds, logbuf := setupDataStore(config.DataStoreConfig{MailboxMsgCap: mbCap})
+	defer teardownDataStore(ds)
+
+	mbName := "captain"
+	for i := 0; i < 20; i++ {
+		// Add a message
+		subj := fmt.Sprintf("subject %v", i)
+		deliverMessage(ds, mbName, subj, time.Now())
+		t.Logf("Delivered %q", subj)
+
+		// Check number of messages
+		mb, err := ds.MailboxFor(mbName)
+		if err != nil {
+			t.Fatalf("Failed to MailboxFor(%q): %v", mbName, err)
+		}
+		msgs, err := mb.GetMessages()
+		if err != nil {
+			t.Fatalf("Failed to GetMessages for %q: %v", mbName, err)
+		}
+		if len(msgs) != i+1 {
+			t.Errorf("Expected %v messages, got %v", i+1, len(msgs))
+		}
+	}
+
+	if t.Failed() {
+		// Wait for handler to finish logging
+		time.Sleep(2 * time.Second)
+		// Dump buffered log data if there was a failure
+		_, _ = io.Copy(os.Stderr, logbuf)
+	}
+}
+
+// Test Get the latest message
+func TestGetLatestMessage(t *testing.T) {
+	ds, logbuf := setupDataStore(config.DataStoreConfig{})
+	defer teardownDataStore(ds)
+
+	// james hashes to 474ba67bdb289c6263b36dfd8a7bed6c85b04943
+	mbName := "james"
+
+	// Test empty mailbox
+	mb, err := ds.MailboxFor(mbName)
+	assert.Nil(t, err)
+	msg, err := mb.GetMessage("latest")
+	assert.Nil(t, msg)
+	assert.Error(t, err)
+
+	// Deliver test message
+	deliverMessage(ds, mbName, "test", time.Now())
+
+	// Deliver test message 2
+	id2, _ := deliverMessage(ds, mbName, "test 2", time.Now())
+
+	// Test get the latest message
+	mb, err = ds.MailboxFor(mbName)
+	assert.Nil(t, err)
+	msg, err = mb.GetMessage("latest")
+	assert.Nil(t, err)
+	assert.True(t, msg.ID() == id2, "Expected %q to be equal to %q", msg.ID(), id2)
+
+	// Deliver test message 3
+	id3, _ := deliverMessage(ds, mbName, "test 3", time.Now())
+
+	mb, err = ds.MailboxFor(mbName)
+	assert.Nil(t, err)
+	msg, err = mb.GetMessage("latest")
+	assert.Nil(t, err)
+	assert.True(t, msg.ID() == id3, "Expected %q to be equal to %q", msg.ID(), id3)
+
+	// Test wrong id
+	_, err = mb.GetMessage("wrongid")
+	assert.Error(t, err)
+
+	if t.Failed() {
+		// Wait for handler to finish logging
+		time.Sleep(2 * time.Second)
+		// Dump buffered log data if there was a failure
+		_, _ = io.Copy(os.Stderr, logbuf)
+	}
+}
+
+// setupDataStore creates a new FileDataStore in a temporary directory
+func setupDataStore(cfg config.DataStoreConfig) (*FileDataStore, *bytes.Buffer) {
+	path, err := ioutil.TempDir("", "inbucket")
+	if err != nil {
+		panic(err)
+	}
+
+	// Capture log output
+	buf := new(bytes.Buffer)
+	log.SetOutput(buf)
+
+	cfg.Path = path
+	return NewFileDataStore(cfg).(*FileDataStore), buf
+}
+
+// deliverMessage creates and delivers a message to the specific mailbox, returning
+// the size of the generated message.
+func deliverMessage(ds *FileDataStore, mbName string, subject string,
+	date time.Time) (id string, size int64) {
+	// Build fake SMTP message for delivery
+	testMsg := make([]byte, 0, 300)
+	testMsg = append(testMsg, []byte("To: somebody@host\r\n")...)
+	testMsg = append(testMsg, []byte("From: somebodyelse@host\r\n")...)
+	testMsg = append(testMsg, []byte(fmt.Sprintf("Subject: %s\r\n", subject))...)
+	testMsg = append(testMsg, []byte("\r\n")...)
+	testMsg = append(testMsg, []byte("Test Body\r\n")...)
+
+	mb, err := ds.MailboxFor(mbName)
+	if err != nil {
+		panic(err)
+	}
+	// Create message object
+	id = generateID(date)
+	msg, err := mb.NewMessage()
+	if err != nil {
+		panic(err)
+	}
+	fmsg := msg.(*FileMessage)
+	fmsg.Fdate = date
+	fmsg.Fid = id
+	if err = msg.Append(testMsg); err != nil {
+		panic(err)
+	}
+	if err = msg.Close(); err != nil {
+		panic(err)
+	}
+
+	return id, int64(len(testMsg))
+}
+
+func teardownDataStore(ds *FileDataStore) {
+	if err := os.RemoveAll(ds.path); err != nil {
+		panic(err)
+	}
+}
+
+func isPresent(path string) bool {
+	_, err := os.Lstat(path)
+	return err == nil
+}
+
+func isFile(path string) bool {
+	if fi, err := os.Lstat(path); err == nil {
+		return !fi.IsDir()
+	}
+	return false
+}
+
+func isDir(path string) bool {
+	if fi, err := os.Lstat(path); err == nil {
+		return fi.IsDir()
+	}
+	return false
+}