From f68f07d896a677b4709e1c4be9b07364ff7c385b Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sat, 20 Oct 2018 20:33:27 -0700 Subject: [PATCH 1/2] file: pool index readers to reduce allocs for #122 --- pkg/storage/file/fstore.go | 33 ++++++++++++++++++++++++++++----- pkg/storage/file/mbox.go | 4 +++- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index a52bc36..2eb51c9 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "os" "path/filepath" + "sync" "time" "github.com/jhillyerd/inbucket/pkg/config" @@ -40,10 +41,11 @@ func countGenerator(c chan int) { // Store implements DataStore aand is the root of the mail storage // hiearchy. It provides access to Mailbox objects type Store struct { - hashLock storage.HashLock - path string - mailPath string - messageCap int + hashLock storage.HashLock + path string + mailPath string + messageCap int + bufReaderPool sync.Pool } // New creates a new DataStore object using the specified path @@ -60,7 +62,16 @@ func New(cfg config.Storage) (storage.Store, error) { Msg("Error creating dir") } } - return &Store{path: path, mailPath: mailPath, messageCap: cfg.MailboxMsgCap}, nil + return &Store{ + path: path, + mailPath: mailPath, + messageCap: cfg.MailboxMsgCap, + bufReaderPool: sync.Pool{ + New: func() interface{} { + return bufio.NewReader(nil) + }, + }, + }, nil } // AddMessage adds a message to the specified mailbox. @@ -253,6 +264,18 @@ func (fs *Store) mboxFromHash(hash string) *mbox { } } +// getPooledReader pulls a buffered reader from the fs.bufReaderPool +func (fs *Store) getPooledReader(r io.Reader) *bufio.Reader { + br := fs.bufReaderPool.Get().(*bufio.Reader) + br.Reset(r) + return br +} + +// putPooledReader returns a buffered reader to the fs.bufReaderPool +func (fs *Store) putPooledReader(br *bufio.Reader) { + fs.bufReaderPool.Put(br) +} + // generatePrefix converts a Time object into the ISO style format we use // as a prefix for message files. Note: It is used directly by unit // tests. diff --git a/pkg/storage/file/mbox.go b/pkg/storage/file/mbox.go index 8c85145..55f1c9f 100644 --- a/pkg/storage/file/mbox.go +++ b/pkg/storage/file/mbox.go @@ -120,7 +120,9 @@ func (mb *mbox) readIndex() error { } }() // Decode gob data - dec := gob.NewDecoder(bufio.NewReader(file)) + br := mb.store.getPooledReader(file) + defer mb.store.putPooledReader(br) + dec := gob.NewDecoder(br) name := "" if err = dec.Decode(&name); err != nil { return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) From 0640f9fa08febf79068afe52f02889a15821a3cf Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Sun, 21 Oct 2018 09:25:32 -0700 Subject: [PATCH 2/2] file: Use os.Readdirnames to eliminate Lstat calls for #122 - This a speed/syscall optimization, not memory. --- pkg/storage/file/fstore.go | 64 +++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/pkg/storage/file/fstore.go b/pkg/storage/file/fstore.go index 2eb51c9..5d3dcc7 100644 --- a/pkg/storage/file/fstore.go +++ b/pkg/storage/file/fstore.go @@ -4,7 +4,6 @@ import ( "bufio" "fmt" "io" - "io/ioutil" "os" "path/filepath" "sync" @@ -190,41 +189,33 @@ func (fs *Store) PurgeMessages(mailbox string) error { // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it // continues to return true. func (fs *Store) VisitMailboxes(f func([]storage.Message) (cont bool)) error { - infos1, err := ioutil.ReadDir(fs.mailPath) + names1, err := readDirNames(fs.mailPath) if err != nil { return err } // Loop over level 1 directories - for _, inf1 := range infos1 { - if inf1.IsDir() { - l1 := inf1.Name() - infos2, err := ioutil.ReadDir(filepath.Join(fs.mailPath, l1)) + for _, name1 := range names1 { + names2, err := readDirNames(fs.mailPath, name1) + if err != nil { + return err + } + // Loop over level 2 directories + for _, name2 := range names2 { + names3, err := readDirNames(fs.mailPath, name1, name2) if err != nil { return err } - // Loop over level 2 directories - for _, inf2 := range infos2 { - if inf2.IsDir() { - l2 := inf2.Name() - infos3, err := ioutil.ReadDir(filepath.Join(fs.mailPath, l1, l2)) - if err != nil { - return err - } - // Loop over mailboxes - for _, inf3 := range infos3 { - if inf3.IsDir() { - mb := fs.mboxFromHash(inf3.Name()) - mb.RLock() - msgs, err := mb.getMessages() - mb.RUnlock() - if err != nil { - return err - } - if !f(msgs) { - return nil - } - } - } + // Loop over mailboxes + for _, name3 := range names3 { + mb := fs.mboxFromHash(name3) + mb.RLock() + msgs, err := mb.getMessages() + mb.RUnlock() + if err != nil { + return err + } + if !f(msgs) { + return nil } } } @@ -264,14 +255,14 @@ func (fs *Store) mboxFromHash(hash string) *mbox { } } -// getPooledReader pulls a buffered reader from the fs.bufReaderPool +// getPooledReader pulls a buffered reader from the fs.bufReaderPool. func (fs *Store) getPooledReader(r io.Reader) *bufio.Reader { br := fs.bufReaderPool.Get().(*bufio.Reader) br.Reset(r) return br } -// putPooledReader returns a buffered reader to the fs.bufReaderPool +// putPooledReader returns a buffered reader to the fs.bufReaderPool. func (fs *Store) putPooledReader(br *bufio.Reader) { fs.bufReaderPool.Put(br) } @@ -284,7 +275,16 @@ func generatePrefix(date time.Time) string { } // generateId adds a 4-digit unique number onto the end of the string -// returned by generatePrefix() +// returned by generatePrefix(). func generateID(date time.Time) string { return generatePrefix(date) + "-" + fmt.Sprintf("%04d", <-countChannel) } + +// readDirNames returns a slice of filenames in the specified directory or an error. +func readDirNames(elem ...string) ([]string, error) { + f, err := os.Open(filepath.Join(elem...)) + if err != nil { + return nil, err + } + return f.Readdirnames(0) +}