package file import ( "bufio" "encoding/gob" "fmt" "io" "io/ioutil" "os" "path/filepath" "sync" "time" "github.com/jhillyerd/inbucket/pkg/config" "github.com/jhillyerd/inbucket/pkg/log" "github.com/jhillyerd/inbucket/pkg/storage" "github.com/jhillyerd/inbucket/pkg/stringutil" ) // Name of index file in each mailbox const indexFileName = "index.gob" var ( // indexMx is locked while reading/writing an index file // // NOTE: This is a bottleneck because it's a single lock even if we have a // million index files indexMx = new(sync.RWMutex) // dirMx is locked while creating/removing directories dirMx = new(sync.Mutex) // countChannel is filled with a sequential numbers (0000..9999), which are // used by generateID() to generate unique message IDs. It's global // because we only want one regardless of the number of DataStore objects countChannel = make(chan int, 10) ) func init() { // Start generator go countGenerator(countChannel) } // Populates the channel with numbers func countGenerator(c chan int) { for i := 0; true; i = (i + 1) % 10000 { c <- i } } // Store implements DataStore aand is the root of the mail storage // hiearchy. It provides access to Mailbox objects type Store struct { hashLock storage.HashLock path string mailPath string messageCap int } // New creates a new DataStore object using the specified path func New(cfg config.DataStoreConfig) storage.Store { path := cfg.Path if path == "" { log.Errorf("No value configured for datastore path") return nil } mailPath := filepath.Join(path, "mail") if _, err := os.Stat(mailPath); err != nil { // Mail datastore does not yet exist if err = os.MkdirAll(mailPath, 0770); err != nil { log.Errorf("Error creating dir %q: %v", mailPath, err) } } return &Store{path: path, mailPath: mailPath, messageCap: cfg.MailboxMsgCap} } // GetMessage returns the messages in the named mailbox, or an error. func (fs *Store) GetMessage(mailbox, id string) (storage.StoreMessage, error) { mb, err := fs.mbox(mailbox) if err != nil { return nil, err } return mb.getMessage(id) } // GetMessages returns the messages in the named mailbox, or an error. func (fs *Store) GetMessages(mailbox string) ([]storage.StoreMessage, error) { mb, err := fs.mbox(mailbox) if err != nil { return nil, err } return mb.getMessages() } // RemoveMessage deletes a message by ID from the specified mailbox. func (fs *Store) RemoveMessage(mailbox, id string) error { mb, err := fs.mbox(mailbox) if err != nil { return err } return mb.removeMessage(id) } // PurgeMessages deletes all messages in the named mailbox, or returns an error. func (fs *Store) PurgeMessages(mailbox string) error { mb, err := fs.mbox(mailbox) if err != nil { return err } return mb.purge() } // VisitMailboxes accepts a function that will be called with the messages in each mailbox while it // continues to return true. func (fs *Store) VisitMailboxes(f func([]storage.StoreMessage) (cont bool)) error { infos1, err := ioutil.ReadDir(fs.mailPath) if err != nil { return err } // Loop over level 1 directories for _, inf1 := range infos1 { if inf1.IsDir() { l1 := inf1.Name() infos2, err := ioutil.ReadDir(filepath.Join(fs.mailPath, l1)) if err != nil { return err } // Loop over level 2 directories for _, inf2 := range infos2 { if inf2.IsDir() { l2 := inf2.Name() infos3, err := ioutil.ReadDir(filepath.Join(fs.mailPath, l1, l2)) if err != nil { return err } // Loop over mailboxes for _, inf3 := range infos3 { if inf3.IsDir() { mbdir := inf3.Name() mbpath := filepath.Join(fs.mailPath, l1, l2, mbdir) idx := filepath.Join(mbpath, indexFileName) mb := &mbox{store: fs, dirName: mbdir, path: mbpath, indexPath: idx} msgs, err := mb.getMessages() if err != nil { return err } if !f(msgs) { return nil } } } } } } } return nil } // LockFor returns the RWMutex for this mailbox, or an error. func (fs *Store) LockFor(emailAddress string) (*sync.RWMutex, error) { name, err := stringutil.ParseMailboxName(emailAddress) if err != nil { return nil, err } hash := stringutil.HashMailboxName(name) return fs.hashLock.Get(hash), nil } // NewMessage is temproary until #69 MessageData refactor func (fs *Store) NewMessage(mailbox string) (storage.StoreMessage, error) { mb, err := fs.mbox(mailbox) if err != nil { return nil, err } return mb.newMessage() } // mbox returns the named mailbox. func (fs *Store) mbox(mailbox string) (*mbox, error) { name, err := stringutil.ParseMailboxName(mailbox) if err != nil { return nil, err } dir := stringutil.HashMailboxName(name) s1 := dir[0:3] s2 := dir[0:6] path := filepath.Join(fs.mailPath, s1, s2, dir) indexPath := filepath.Join(path, indexFileName) return &mbox{store: fs, name: name, dirName: dir, path: path, indexPath: indexPath}, nil } // mbox manages the mail for a specific user and correlates to a particular directory on disk. type mbox struct { store *Store name string dirName string path string indexLoaded bool indexPath string messages []*Message } // getMessages scans the mailbox directory for .gob files and decodes them into // a slice of Message objects. func (mb *mbox) getMessages() ([]storage.StoreMessage, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err } } messages := make([]storage.StoreMessage, len(mb.messages)) for i, m := range mb.messages { messages[i] = m } return messages, nil } // getMessage decodes a single message by ID and returns a Message object. func (mb *mbox) getMessage(id string) (storage.StoreMessage, error) { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return nil, err } } if id == "latest" && len(mb.messages) != 0 { return mb.messages[len(mb.messages)-1], nil } for _, m := range mb.messages { if m.Fid == id { return m, nil } } return nil, storage.ErrNotExist } // removeMessage deletes the message off disk and removes it from the index. func (mb *mbox) removeMessage(id string) error { if !mb.indexLoaded { if err := mb.readIndex(); err != nil { return err } } var msg *Message for i, m := range mb.messages { if id == m.ID() { msg = m // Slice around message we are deleting mb.messages = append(mb.messages[:i], mb.messages[i+1:]...) break } } if msg == nil { return storage.ErrNotExist } if err := mb.writeIndex(); err != nil { return err } if len(mb.messages) == 0 { // This was the last message, thus writeIndex() has removed the entire // directory; we don't need to delete the raw file. return nil } // There are still messages in the index log.Tracef("Deleting %v", msg.rawPath()) return os.Remove(msg.rawPath()) } // purge deletes all messages in this mailbox. func (mb *mbox) purge() error { mb.messages = mb.messages[:0] return mb.writeIndex() } // readIndex loads the mailbox index data from disk func (mb *mbox) readIndex() error { // Clear message slice, open index mb.messages = mb.messages[:0] // Lock for reading indexMx.RLock() defer indexMx.RUnlock() // Check if index exists if _, err := os.Stat(mb.indexPath); err != nil { // Does not exist, but that's not an error in our world log.Tracef("Index %v does not exist (yet)", mb.indexPath) mb.indexLoaded = true return nil } file, err := os.Open(mb.indexPath) if err != nil { return err } defer func() { if err := file.Close(); err != nil { log.Errorf("Failed to close %q: %v", mb.indexPath, err) } }() // Decode gob data dec := gob.NewDecoder(bufio.NewReader(file)) name := "" if err = dec.Decode(&name); err != nil { return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) } mb.name = name for { // Load messages until EOF msg := &Message{} if err = dec.Decode(msg); err != nil { if err == io.EOF { break } return fmt.Errorf("Corrupt mailbox %q: %v", mb.indexPath, err) } msg.mailbox = mb mb.messages = append(mb.messages, msg) } mb.indexLoaded = true return nil } // writeIndex overwrites the index on disk with the current mailbox data func (mb *mbox) writeIndex() error { // Lock for writing indexMx.Lock() defer indexMx.Unlock() if len(mb.messages) > 0 { // Ensure mailbox directory exists if err := mb.createDir(); err != nil { return err } // Open index for writing file, err := os.Create(mb.indexPath) if err != nil { return err } writer := bufio.NewWriter(file) // Write each message and then flush enc := gob.NewEncoder(writer) if err = enc.Encode(mb.name); err != nil { _ = file.Close() return err } for _, m := range mb.messages { if err = enc.Encode(m); err != nil { _ = file.Close() return err } } if err := writer.Flush(); err != nil { _ = file.Close() return err } if err := file.Close(); err != nil { log.Errorf("Failed to close %q: %v", mb.indexPath, err) return err } } else { // No messages, delete index+maildir log.Tracef("Removing mailbox %v", mb.path) return mb.removeDir() } return nil } // createDir checks for the presence of the path for this mailbox, creates it if needed func (mb *mbox) createDir() error { dirMx.Lock() defer dirMx.Unlock() if _, err := os.Stat(mb.path); err != nil { if err := os.MkdirAll(mb.path, 0770); err != nil { log.Errorf("Failed to create directory %v, %v", mb.path, err) return err } } return nil } // removeDir removes the mailbox, plus empty higher level directories func (mb *mbox) removeDir() error { dirMx.Lock() defer dirMx.Unlock() // remove mailbox dir, including index file if err := os.RemoveAll(mb.path); err != nil { return err } // remove parents if empty dir := filepath.Dir(mb.path) if removeDirIfEmpty(dir) { removeDirIfEmpty(filepath.Dir(dir)) } return nil } // removeDirIfEmpty will remove the specified directory if it contains no files or directories. // Caller should hold dirMx. Returns true if dir was removed. func removeDirIfEmpty(path string) (removed bool) { f, err := os.Open(path) if err != nil { return false } files, err := f.Readdirnames(0) _ = f.Close() if err != nil { return false } if len(files) > 0 { // Dir not empty return false } log.Tracef("Removing dir %v", path) err = os.Remove(path) if err != nil { log.Errorf("Failed to remove %q: %v", path, err) return false } return true } // generatePrefix converts a Time object into the ISO style format we use // as a prefix for message files. Note: It is used directly by unit // tests. func generatePrefix(date time.Time) string { return date.Format("20060102T150405") } // generateId adds a 4-digit unique number onto the end of the string // returned by generatePrefix() func generateID(date time.Time) string { return generatePrefix(date) + "-" + fmt.Sprintf("%04d", <-countChannel) }