1
0
mirror of https://github.com/jhillyerd/inbucket.git synced 2026-01-25 12:35:55 +00:00

Reorganize packages pt 1

End goal: simplify build process
This commit is contained in:
James Hillyerd
2012-10-22 15:20:33 -07:00
parent 21a9211c0f
commit 4e5c0ce4d8
13 changed files with 53 additions and 47 deletions

302
smtpd/datastore.go Normal file
View File

@@ -0,0 +1,302 @@
package smtpd
import (
"bufio"
"encoding/gob"
"errors"
"fmt"
"github.com/jhillyerd/inbucket"
"github.com/jhillyerd/inbucket/log"
"io/ioutil"
"net/mail"
"os"
"path/filepath"
"strings"
"time"
)
var ErrNotWritable = errors.New("Message not writable")
// Global because we only want one regardless of the number of DataStore objects
var countChannel = make(chan int, 10)
func init() {
// Start generator
go countGenerator(countChannel)
}
// Populates the channel with numbers
func countGenerator(c chan int) {
for i := 0; true; i = (i + 1) % 10000 {
c <- i
}
}
// A DataStore is the root of the mail storage hiearchy. It provides access to
// Mailbox objects
type DataStore struct {
path string
mailPath string
}
// NewDataStore creates a new DataStore object. It uses the inbucket.Config object to
// construct it's path.
func NewDataStore() *DataStore {
path, err := inbucket.Config.String("datastore", "path")
if err != nil {
log.Error("Error getting datastore path: %v", err)
return nil
}
if path == "" {
log.Error("No value configured for datastore path")
return nil
}
mailPath := filepath.Join(path, "mail")
return &DataStore{path: path, mailPath: mailPath}
}
// Retrieves the Mailbox object for a specified email address, if the mailbox
// does not exist, it will attempt to create it.
func (ds *DataStore) MailboxFor(emailAddress string) (*Mailbox, error) {
name := ParseMailboxName(emailAddress)
dir := HashMailboxName(name)
path := filepath.Join(ds.mailPath, dir)
if err := os.MkdirAll(path, 0770); err != nil {
log.Error("Failed to create directory %v, %v", path, err)
return nil, err
}
return &Mailbox{store: ds, name: name, dirName: dir, path: path}, nil
}
// A Mailbox manages the mail for a specific user and correlates to a particular
// directory on disk.
type Mailbox struct {
store *DataStore
name string
dirName string
path string
}
func (mb *Mailbox) String() string {
return mb.name + "[" + mb.dirName + "]"
}
// GetMessages scans the mailbox directory for .gob files and decodes them into
// a slice of Message objects.
func (mb *Mailbox) GetMessages() ([]*Message, error) {
files, err := ioutil.ReadDir(mb.path)
if err != nil {
return nil, err
}
log.Trace("Scanning %v files for %v", len(files), mb)
messages := make([]*Message, 0, len(files))
for _, f := range files {
if (!f.IsDir()) && strings.HasSuffix(strings.ToLower(f.Name()), ".gob") {
// We have a gob file
file, err := os.Open(filepath.Join(mb.path, f.Name()))
if err != nil {
return nil, err
}
dec := gob.NewDecoder(bufio.NewReader(file))
msg := new(Message)
if err = dec.Decode(msg); err != nil {
return nil, err
}
file.Close()
msg.mailbox = mb
log.Trace("Found: %v", msg)
messages = append(messages, msg)
}
}
return messages, nil
}
// GetMessage decodes a single message by Id and returns a Message object
func (mb *Mailbox) GetMessage(id string) (*Message, error) {
file, err := os.Open(filepath.Join(mb.path, id+".gob"))
if err != nil {
return nil, err
}
dec := gob.NewDecoder(bufio.NewReader(file))
msg := new(Message)
if err = dec.Decode(msg); err != nil {
return nil, err
}
file.Close()
msg.mailbox = mb
log.Trace("Found: %v", msg)
return msg, nil
}
// Message contains a little bit of data about a particular email message, and
// methods to retrieve the rest of it from disk.
type Message struct {
mailbox *Mailbox
Id string
Date time.Time
From string
Subject string
// These are for creating new messages only
writable bool
writerFile *os.File
writer *bufio.Writer
}
// NewMessage creates a new Message object and sets the Date and Id fields.
func (mb *Mailbox) NewMessage() *Message {
date := time.Now()
id := date.Format("20060102T150405") + "-" + fmt.Sprintf("%04d", <-countChannel)
return &Message{mailbox: mb, Id: id, Date: date, writable: true}
}
func (m *Message) String() string {
return fmt.Sprintf("\"%v\" from %v", m.Subject, m.From)
}
func (m *Message) gobPath() string {
return filepath.Join(m.mailbox.path, m.Id+".gob")
}
func (m *Message) rawPath() string {
return filepath.Join(m.mailbox.path, m.Id+".raw")
}
// ReadHeader opens the .raw portion of a Message and returns a standard Go mail.Message object
func (m *Message) ReadHeader() (msg *mail.Message, err error) {
file, err := os.Open(m.rawPath())
defer file.Close()
if err != nil {
return nil, err
}
reader := bufio.NewReader(file)
msg, err = mail.ReadMessage(reader)
return msg, err
}
// ReadBody opens the .raw portion of a Message and returns a MIMEBody object, along
// with a free mail.Message containing the Headers, since we had to make one of those
// anyway.
func (m *Message) ReadBody() (msg *mail.Message, body *MIMEBody, err error) {
file, err := os.Open(m.rawPath())
defer file.Close()
if err != nil {
return nil, nil, err
}
reader := bufio.NewReader(file)
msg, err = mail.ReadMessage(reader)
if err != nil {
return nil, nil, err
}
mime, err := ParseMIMEBody(msg)
if err != nil {
return nil, nil, err
}
return msg, mime, err
}
// ReadRaw opens the .raw portion of a Message and returns it as a string
func (m *Message) ReadRaw() (raw *string, err error) {
file, err := os.Open(m.rawPath())
defer file.Close()
if err != nil {
return nil, err
}
reader := bufio.NewReader(file)
bodyBytes, err := ioutil.ReadAll(reader)
if err != nil {
return nil, err
}
bodyString := string(bodyBytes)
return &bodyString, nil
}
// Append data to a newly opened Message, this will fail on a pre-existing Message and
// after Close() is called.
func (m *Message) Append(data []byte) error {
// Prevent Appending to a pre-existing Message
if !m.writable {
return ErrNotWritable
}
// Open file for writing if we haven't yet
if m.writer == nil {
file, err := os.Create(m.rawPath())
if err != nil {
// Set writable false just in case something calls me a million times
m.writable = false
return err
}
m.writerFile = file
m.writer = bufio.NewWriter(file)
}
_, err := m.writer.Write(data)
return err
}
// Close this Message for writing - no more data may be Appended. Close() will also
// trigger the creation of the .gob file.
func (m *Message) Close() error {
// nil out the writer fields so they can't be used
writer := m.writer
writerFile := m.writerFile
m.writer = nil
m.writerFile = nil
if writer != nil {
if err := writer.Flush(); err != nil {
return err
}
}
if writerFile != nil {
if err := writerFile.Close(); err != nil {
return err
}
}
return m.createGob()
}
// Delete this Message from disk by removing both the gob and raw files
func (m *Message) Delete() error {
log.Trace("Deleting %v", m.gobPath())
err := os.Remove(m.gobPath())
if err != nil {
return err
}
log.Trace("Deleting %v", m.rawPath())
return os.Remove(m.rawPath())
}
// createGob reads the .raw file to grab the From and Subject header entries,
// then creates the .gob file.
func (m *Message) createGob() error {
// Open gob for writing
file, err := os.Create(m.gobPath())
defer file.Close()
if err != nil {
return err
}
writer := bufio.NewWriter(file)
// Fetch headers
msg, err := m.ReadHeader()
if err != nil {
return err
}
// Only public fields are stored in gob
m.From = msg.Header.Get("From")
m.Subject = msg.Header.Get("Subject")
// Write & flush
enc := gob.NewEncoder(writer)
err = enc.Encode(m)
if err != nil {
return err
}
writer.Flush()
return nil
}

View File

@@ -5,7 +5,7 @@ import (
"bytes"
"container/list"
"fmt"
"github.com/jhillyerd/inbucket"
"github.com/jhillyerd/inbucket/log"
"net"
"regexp"
"strconv"
@@ -87,7 +87,7 @@ func (ss *Session) String() string {
* 5. Goto 2
*/
func (s *Server) startSession(id int, conn net.Conn) {
inbucket.Info("Connection from %v, starting session <%v>", conn.RemoteAddr(), id)
log.Info("Connection from %v, starting session <%v>", conn.RemoteAddr(), id)
defer conn.Close()
ss := NewSession(s, id, conn)
@@ -279,8 +279,8 @@ func (ss *Session) dataHandler() {
msgSize := 0
// Get a Mailbox and a new Message for each recipient
mailboxes := make([]*inbucket.Mailbox, ss.recipients.Len())
messages := make([]*inbucket.Message, ss.recipients.Len())
mailboxes := make([]*Mailbox, ss.recipients.Len())
messages := make([]*Message, ss.recipients.Len())
i := 0
for e := ss.recipients.Front(); e != nil; e = e.Next() {
recip := e.Value.(string)
@@ -476,17 +476,17 @@ func (ss *Session) ooSeq(cmd string) {
// Session specific logging methods
func (ss *Session) trace(msg string, args ...interface{}) {
inbucket.Trace("%v<%v> %v", ss.remoteHost, ss.id, fmt.Sprintf(msg, args...))
log.Trace("%v<%v> %v", ss.remoteHost, ss.id, fmt.Sprintf(msg, args...))
}
func (ss *Session) info(msg string, args ...interface{}) {
inbucket.Info("%v<%v> %v", ss.remoteHost, ss.id, fmt.Sprintf(msg, args...))
log.Info("%v<%v> %v", ss.remoteHost, ss.id, fmt.Sprintf(msg, args...))
}
func (ss *Session) warn(msg string, args ...interface{}) {
inbucket.Warn("%v<%v> %v", ss.remoteHost, ss.id, fmt.Sprintf(msg, args...))
log.Warn("%v<%v> %v", ss.remoteHost, ss.id, fmt.Sprintf(msg, args...))
}
func (ss *Session) error(msg string, args ...interface{}) {
inbucket.Error("%v<%v> %v", ss.remoteHost, ss.id, fmt.Sprintf(msg, args...))
log.Error("%v<%v> %v", ss.remoteHost, ss.id, fmt.Sprintf(msg, args...))
}

View File

@@ -3,6 +3,7 @@ package smtpd
import (
"fmt"
"github.com/jhillyerd/inbucket"
"github.com/jhillyerd/inbucket/log"
"net"
)
@@ -12,12 +13,12 @@ type Server struct {
maxRecips int
maxIdleSeconds int
maxMessageBytes int
dataStore *inbucket.DataStore
dataStore *DataStore
}
// Init a new Server object
func New() *Server {
ds := inbucket.NewDataStore()
ds := NewDataStore()
// TODO Make more of these configurable
return &Server{domain: inbucket.GetSmtpConfig().Domain, maxRecips: 100, maxIdleSeconds: 300,
dataStore: ds, maxMessageBytes: 2048000}
@@ -29,15 +30,15 @@ func (s *Server) Start() {
addr, err := net.ResolveTCPAddr("tcp4", fmt.Sprintf("%v:%v",
cfg.Ip4address, cfg.Ip4port))
if err != nil {
inbucket.Error("Failed to build tcp4 address: %v", err)
log.Error("Failed to build tcp4 address: %v", err)
// TODO More graceful early-shutdown procedure
panic(err)
}
inbucket.Info("SMTP listening on TCP4 %v", addr)
log.Info("SMTP listening on TCP4 %v", addr)
ln, err := net.ListenTCP("tcp4", addr)
if err != nil {
inbucket.Error("Failed to start tcp4 listener: %v", err)
log.Error("Failed to start tcp4 listener: %v", err)
// TODO More graceful early-shutdown procedure
panic(err)
}

209
smtpd/mime.go Normal file
View File

@@ -0,0 +1,209 @@
package smtpd
import (
"bytes"
"container/list"
"fmt"
"github.com/sloonz/go-qprintable"
"io"
"mime"
"mime/multipart"
"net/mail"
"strings"
)
type MIMENodeMatcher func(node *MIMENode) bool
type MIMENode struct {
Parent *MIMENode
FirstChild *MIMENode
NextSibling *MIMENode
Type string
Content []byte
}
type MIMEBody struct {
Text string
Html string
Root *MIMENode
}
func NewMIMENode(parent *MIMENode, contentType string) *MIMENode {
return &MIMENode{Parent: parent, Type: contentType}
}
func (n *MIMENode) BreadthFirstSearch(matcher MIMENodeMatcher) *MIMENode {
q := list.New()
q.PushBack(n)
// Push children onto queue and attempt to match in that order
for q.Len() > 0 {
e := q.Front()
n := e.Value.(*MIMENode)
if matcher(n) {
return n
}
q.Remove(e)
c := n.FirstChild
for c != nil {
q.PushBack(c)
c = c.NextSibling
}
}
return nil
}
func (n *MIMENode) String() string {
children := ""
siblings := ""
if n.FirstChild != nil {
children = n.FirstChild.String()
}
if n.NextSibling != nil {
siblings = n.NextSibling.String()
}
return fmt.Sprintf("[%v %v] %v", n.Type, children, siblings)
}
func IsMIMEMessage(mailMsg *mail.Message) bool {
// Parse top-level multipart
ctype := mailMsg.Header.Get("Content-Type")
mediatype, _, err := mime.ParseMediaType(ctype)
if err != nil {
return false
}
switch mediatype {
case "multipart/alternative":
return true
}
return false
}
func ParseMIMEBody(mailMsg *mail.Message) (*MIMEBody, error) {
mimeMsg := new(MIMEBody)
if !IsMIMEMessage(mailMsg) {
// Parse as text only
bodyBytes, err := decodeSection(mailMsg.Header.Get("Content-Transfer-Encoding"), mailMsg.Body)
if err != nil {
return nil, err
}
mimeMsg.Text = string(bodyBytes)
} else {
// Parse top-level multipart
ctype := mailMsg.Header.Get("Content-Type")
mediatype, params, err := mime.ParseMediaType(ctype)
if err != nil {
return nil, err
}
switch mediatype {
case "multipart/alternative":
// Good
default:
return nil, fmt.Errorf("Unknown mediatype: %v", mediatype)
}
boundary := params["boundary"]
if boundary == "" {
return nil, fmt.Errorf("Unable to locate boundary param in Content-Type header")
}
// Root Node of our tree
root := NewMIMENode(nil, mediatype)
err = parseNodes(root, mailMsg.Body, boundary)
// Locate text body
match := root.BreadthFirstSearch(func(node *MIMENode) bool {
return node.Type == "text/plain"
})
if match != nil {
mimeMsg.Text = string(match.Content)
}
// Locate HTML body
match = root.BreadthFirstSearch(func(node *MIMENode) bool {
return node.Type == "text/html"
})
if match != nil {
mimeMsg.Html = string(match.Content)
}
}
return mimeMsg, nil
}
func (m *MIMEBody) String() string {
return fmt.Sprintf("----TEXT----\n%v\n----HTML----\n%v\n----END----\n", m.Text, m.Html)
}
// decodeSection attempts to decode the data from reader using the algorithm listed in
// the Content-Transfer-Encoding header, returning the raw data if it does not know
// the encoding type.
func decodeSection(encoding string, reader io.Reader) ([]byte, error) {
switch strings.ToLower(encoding) {
case "quoted-printable":
decoder := qprintable.NewDecoder(qprintable.WindowsTextEncoding, reader)
buf := new(bytes.Buffer)
_, err := buf.ReadFrom(decoder)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// Don't know this type, just return bytes
buf := new(bytes.Buffer)
_, err := buf.ReadFrom(reader)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func parseNodes(parent *MIMENode, reader io.Reader, boundary string) error {
var prevSibling *MIMENode
// Loop over MIME parts
mr := multipart.NewReader(reader, boundary)
for {
part, err := mr.NextPart()
if err != nil {
if err == io.EOF {
// This is a clean end-of-message signal
break
}
return err
}
mediatype, params, err := mime.ParseMediaType(part.Header.Get("Content-Type"))
if err != nil {
return err
}
// Insert ourselves into tree
node := NewMIMENode(parent, mediatype)
if prevSibling != nil {
prevSibling.NextSibling = node
} else {
parent.FirstChild = node
}
prevSibling = node
boundary := params["boundary"]
if boundary != "" {
// Content is another multipart
err = parseNodes(node, part, boundary)
if err != nil {
return err
}
} else {
// Content is text or data, decode it
data, err := decodeSection(part.Header.Get("Content-Transfer-Encoding"), part)
if err != nil {
return err
}
node.Content = data
}
}
return nil
}

95
smtpd/mime_test.go Normal file
View File

@@ -0,0 +1,95 @@
package smtpd
import (
"bufio"
"fmt"
"github.com/stretchrcom/testify/assert"
"net/mail"
"os"
"path/filepath"
"testing"
)
func TestIdentifyNonMime(t *testing.T) {
msg := readMessage("non-mime.raw")
assert.False(t, IsMIMEMessage(msg), "Failed to identify non-MIME message")
}
func TestIdentifyMime(t *testing.T) {
msg := readMessage("html-mime-inline.raw")
assert.True(t, IsMIMEMessage(msg), "Failed to identify MIME message")
}
func TestParseNonMime(t *testing.T) {
msg := readMessage("non-mime.raw")
mime, err := ParseMIMEBody(msg)
if err != nil {
t.Fatalf("Failed to parse non-MIME: %v", err)
}
assert.Contains(t, mime.Text, "This is a test mailing")
}
func TestParseInlineText(t *testing.T) {
msg := readMessage("html-mime-inline.raw")
mime, err := ParseMIMEBody(msg)
if err != nil {
t.Fatalf("Failed to parse MIME: %v", err)
}
assert.Equal(t, mime.Text, "Test of HTML section")
}
func TestParseQuotedPrintable(t *testing.T) {
msg := readMessage("quoted-printable.raw")
mime, err := ParseMIMEBody(msg)
if err != nil {
t.Fatalf("Failed to parse MIME: %v", err)
}
assert.Contains(t, mime.Text, "Phasellus sit amet arcu")
}
func TestParseQuotedPrintableMime(t *testing.T) {
msg := readMessage("quoted-printable-mime.raw")
mime, err := ParseMIMEBody(msg)
if err != nil {
t.Fatalf("Failed to parse MIME: %v", err)
}
assert.Contains(t, mime.Text, "Nullam venenatis ante")
}
func TestParseInlineHtml(t *testing.T) {
msg := readMessage("html-mime-inline.raw")
mime, err := ParseMIMEBody(msg)
if err != nil {
t.Fatalf("Failed to parse MIME: %v", err)
}
assert.Contains(t, mime.Html, "<html>")
assert.Contains(t, mime.Html, "Test of HTML section")
}
// readMessage is a test utility function to fetch a mail.Message object.
func readMessage(filename string) *mail.Message {
// Open test email for parsing
raw, err := os.Open(filepath.Join("..", "test-data", filename))
if err != nil {
panic(fmt.Sprintf("Failed to open test data: %v", err))
}
// Parse email into a mail.Message object like we do
reader := bufio.NewReader(raw)
msg, err := mail.ReadMessage(reader)
if err != nil {
panic(fmt.Sprintf("Failed to read message: %v", err))
}
return msg
}

36
smtpd/utils.go Normal file
View File

@@ -0,0 +1,36 @@
package smtpd
import (
"crypto/sha1"
"fmt"
"html"
"io"
"strings"
)
// Take "user+ext@host.com" and return "user", aka the mailbox we'll store it in
func ParseMailboxName(emailAddress string) (result string) {
result = strings.ToLower(emailAddress)
if idx := strings.Index(result, "@"); idx > -1 {
result = result[0:idx]
}
if idx := strings.Index(result, "+"); idx > -1 {
result = result[0:idx]
}
return result
}
// Take a mailbox name and hash it into the directory we'll store it in
func HashMailboxName(mailbox string) string {
h := sha1.New()
io.WriteString(h, mailbox)
return fmt.Sprintf("%x", h.Sum(nil))
}
// TextToHtml takes plain text, escapes it and tries to pretty it up for
// HTML display
func TextToHtml(text string) string {
text = html.EscapeString(text)
replacer := strings.NewReplacer("\r\n", "<br/>\n", "\r", "<br/>\n", "\n", "<br/>\n")
return replacer.Replace(text)
}

43
smtpd/utils_test.go Normal file
View File

@@ -0,0 +1,43 @@
package smtpd
import (
"github.com/stretchrcom/testify/assert"
"testing"
)
func TestParseMailboxName(t *testing.T) {
in, out := "MailBOX", "mailbox"
if x := ParseMailboxName(in); x != out {
t.Errorf("ParseMailboxName(%v) = %v, want %v", in, x, out)
}
in, out = "MailBox@Host.Com", "mailbox"
if x := ParseMailboxName(in); x != out {
t.Errorf("ParseMailboxName(%v) = %v, want %v", in, x, out)
}
in, out = "Mail+extra@Host.Com", "mail"
if x := ParseMailboxName(in); x != out {
t.Errorf("ParseMailboxName(%v) = %v, want %v", in, x, out)
}
}
func TestHashMailboxName(t *testing.T) {
in, out := "mail", "1d6e1cf70ec6f9ab28d3ea4b27a49a77654d370e"
if x := HashMailboxName(in); x != out {
t.Errorf("HashMailboxName(%v) = %v, want %v", in, x, out)
}
}
func TestTextToHtml(t *testing.T) {
// Identity
assert.Equal(t, TextToHtml("html"), "html")
// Check it escapes
assert.Equal(t, TextToHtml("<html>"), "&lt;html&gt;")
// Check for linebreaks
assert.Equal(t, TextToHtml("line\nbreak"), "line<br/>\nbreak")
assert.Equal(t, TextToHtml("line\r\nbreak"), "line<br/>\nbreak")
assert.Equal(t, TextToHtml("line\rbreak"), "line<br/>\nbreak")
}