From f62eaa31b97c1e7c0c901ad2890d02d45d9ae655 Mon Sep 17 00:00:00 2001 From: James Hillyerd Date: Tue, 26 Dec 2017 18:33:56 -0800 Subject: [PATCH] Move retention scanner into datastore pkg for #67 --- {smtpd => datastore}/retention.go | 12 +- datastore/retention_test.go | 67 ++++++++++ smtpd/listener.go | 12 +- smtpd/retention_test.go | 198 ------------------------------ 4 files changed, 80 insertions(+), 209 deletions(-) rename {smtpd => datastore}/retention.go (93%) create mode 100644 datastore/retention_test.go delete mode 100644 smtpd/retention_test.go diff --git a/smtpd/retention.go b/datastore/retention.go similarity index 93% rename from smtpd/retention.go rename to datastore/retention.go index a562e5c..b5ede4d 100644 --- a/smtpd/retention.go +++ b/datastore/retention.go @@ -1,4 +1,4 @@ -package smtpd +package datastore import ( "container/list" @@ -7,7 +7,6 @@ import ( "time" "github.com/jhillyerd/inbucket/config" - "github.com/jhillyerd/inbucket/datastore" "github.com/jhillyerd/inbucket/log" ) @@ -37,20 +36,25 @@ func init() { rm.Set("Period", expRetentionPeriod) rm.Set("RetainedHist", expRetainedHist) rm.Set("RetainedCurrent", expRetainedCurrent) + + log.AddTickerFunc(func() { + expRetentionDeletesHist.Set(log.PushMetric(retentionDeletesHist, expRetentionDeletesTotal)) + expRetainedHist.Set(log.PushMetric(retainedHist, expRetainedCurrent)) + }) } // RetentionScanner looks for messages older than the configured retention period and deletes them. type RetentionScanner struct { globalShutdown chan bool // Closes when Inbucket needs to shut down retentionShutdown chan bool // Closed after the scanner has shut down - ds datastore.DataStore + ds DataStore retentionPeriod time.Duration retentionSleep time.Duration } // NewRetentionScanner launches a go-routine that scans for expired // messages, following the configured interval -func NewRetentionScanner(ds datastore.DataStore, shutdownChannel chan bool) *RetentionScanner { +func NewRetentionScanner(ds DataStore, shutdownChannel chan bool) *RetentionScanner { cfg := config.GetDataStoreConfig() rs := &RetentionScanner{ globalShutdown: shutdownChannel, diff --git a/datastore/retention_test.go b/datastore/retention_test.go new file mode 100644 index 0000000..c357f7e --- /dev/null +++ b/datastore/retention_test.go @@ -0,0 +1,67 @@ +package datastore + +import ( + "fmt" + "testing" + "time" +) + +func TestDoRetentionScan(t *testing.T) { + // Create mock objects + mds := &MockDataStore{} + + mb1 := &MockMailbox{} + mb2 := &MockMailbox{} + mb3 := &MockMailbox{} + + // Mockup some different aged messages (num is in hours) + new1 := mockMessage(0) + new2 := mockMessage(1) + new3 := mockMessage(2) + old1 := mockMessage(4) + old2 := mockMessage(12) + old3 := mockMessage(24) + + // First it should ask for all mailboxes + mds.On("AllMailboxes").Return([]Mailbox{mb1, mb2, mb3}, nil) + + // Then for all messages on each box + mb1.On("GetMessages").Return([]Message{new1, old1, old2}, nil) + mb2.On("GetMessages").Return([]Message{old3, new2}, nil) + mb3.On("GetMessages").Return([]Message{new3}, nil) + + // Test 4 hour retention + rs := &RetentionScanner{ + ds: mds, + retentionPeriod: 4*time.Hour - time.Minute, + retentionSleep: 0, + } + if err := rs.doScan(); err != nil { + t.Error(err) + } + + // Check our assertions + mds.AssertExpectations(t) + mb1.AssertExpectations(t) + mb2.AssertExpectations(t) + mb3.AssertExpectations(t) + + // Delete should not have been called on new messages + new1.AssertNotCalled(t, "Delete") + new2.AssertNotCalled(t, "Delete") + new3.AssertNotCalled(t, "Delete") + + // Delete should have been called once on old messages + old1.AssertNumberOfCalls(t, "Delete", 1) + old2.AssertNumberOfCalls(t, "Delete", 1) + old3.AssertNumberOfCalls(t, "Delete", 1) +} + +// Make a MockMessage of a specific age +func mockMessage(ageHours int) *MockMessage { + msg := &MockMessage{} + msg.On("ID").Return(fmt.Sprintf("MSG[age=%vh]", ageHours)) + msg.On("Date").Return(time.Now().Add(time.Duration(ageHours*-1) * time.Hour)) + msg.On("Delete").Return(nil) + return msg +} diff --git a/smtpd/listener.go b/smtpd/listener.go index 06bbf9c..6b4634e 100644 --- a/smtpd/listener.go +++ b/smtpd/listener.go @@ -33,8 +33,6 @@ func init() { expConnectsHist.Set(log.PushMetric(connectsHist, expConnectsTotal)) expErrorsHist.Set(log.PushMetric(errorsHist, expErrorsTotal)) expWarnsHist.Set(log.PushMetric(warnsHist, expWarnsTotal)) - expRetentionDeletesHist.Set(log.PushMetric(retentionDeletesHist, expRetentionDeletesTotal)) - expRetainedHist.Set(log.PushMetric(retainedHist, expRetainedCurrent)) }) } @@ -50,10 +48,10 @@ type Server struct { storeMessages bool // Dependencies - dataStore datastore.DataStore // Mailbox/message store - globalShutdown chan bool // Shuts down Inbucket - msgHub *msghub.Hub // Pub/sub for message info - retentionScanner *RetentionScanner // Deletes expired messages + dataStore datastore.DataStore // Mailbox/message store + globalShutdown chan bool // Shuts down Inbucket + msgHub *msghub.Hub // Pub/sub for message info + retentionScanner *datastore.RetentionScanner // Deletes expired messages // State listener net.Listener // Incoming network connections @@ -98,7 +96,7 @@ func NewServer( globalShutdown: globalShutdown, dataStore: ds, msgHub: msgHub, - retentionScanner: NewRetentionScanner(ds, globalShutdown), + retentionScanner: datastore.NewRetentionScanner(ds, globalShutdown), waitgroup: new(sync.WaitGroup), } } diff --git a/smtpd/retention_test.go b/smtpd/retention_test.go deleted file mode 100644 index e553ac4..0000000 --- a/smtpd/retention_test.go +++ /dev/null @@ -1,198 +0,0 @@ -package smtpd - -import ( - "fmt" - "io" - "net/mail" - "testing" - "time" - - "github.com/jhillyerd/enmime" - "github.com/jhillyerd/inbucket/datastore" - "github.com/stretchr/testify/mock" -) - -func TestDoRetentionScan(t *testing.T) { - // Create mock objects - mds := &MockDataStore{} - - mb1 := &MockMailbox{} - mb2 := &MockMailbox{} - mb3 := &MockMailbox{} - - // Mockup some different aged messages (num is in hours) - new1 := mockMessage(0) - new2 := mockMessage(1) - new3 := mockMessage(2) - old1 := mockMessage(4) - old2 := mockMessage(12) - old3 := mockMessage(24) - - // First it should ask for all mailboxes - mds.On("AllMailboxes").Return([]datastore.Mailbox{mb1, mb2, mb3}, nil) - - // Then for all messages on each box - mb1.On("GetMessages").Return([]datastore.Message{new1, old1, old2}, nil) - mb2.On("GetMessages").Return([]datastore.Message{old3, new2}, nil) - mb3.On("GetMessages").Return([]datastore.Message{new3}, nil) - - // Test 4 hour retention - rs := &RetentionScanner{ - ds: mds, - retentionPeriod: 4*time.Hour - time.Minute, - retentionSleep: 0, - } - if err := rs.doScan(); err != nil { - t.Error(err) - } - - // Check our assertions - mds.AssertExpectations(t) - mb1.AssertExpectations(t) - mb2.AssertExpectations(t) - mb3.AssertExpectations(t) - - // Delete should not have been called on new messages - new1.AssertNotCalled(t, "Delete") - new2.AssertNotCalled(t, "Delete") - new3.AssertNotCalled(t, "Delete") - - // Delete should have been called once on old messages - old1.AssertNumberOfCalls(t, "Delete", 1) - old2.AssertNumberOfCalls(t, "Delete", 1) - old3.AssertNumberOfCalls(t, "Delete", 1) -} - -// Make a MockMessage of a specific age -func mockMessage(ageHours int) *MockMessage { - msg := &MockMessage{} - msg.On("ID").Return(fmt.Sprintf("MSG[age=%vh]", ageHours)) - msg.On("Date").Return(time.Now().Add(time.Duration(ageHours*-1) * time.Hour)) - msg.On("Delete").Return(nil) - return msg -} - -// Mock DataStore object -type MockDataStore struct { - mock.Mock -} - -func (m *MockDataStore) MailboxFor(name string) (datastore.Mailbox, error) { - args := m.Called(name) - return args.Get(0).(datastore.Mailbox), args.Error(1) -} - -func (m *MockDataStore) AllMailboxes() ([]datastore.Mailbox, error) { - args := m.Called() - return args.Get(0).([]datastore.Mailbox), args.Error(1) -} - -// Mock Mailbox object -type MockMailbox struct { - mock.Mock -} - -func (m *MockMailbox) GetMessages() ([]datastore.Message, error) { - args := m.Called() - return args.Get(0).([]datastore.Message), args.Error(1) -} - -func (m *MockMailbox) GetMessage(id string) (datastore.Message, error) { - args := m.Called(id) - return args.Get(0).(datastore.Message), args.Error(1) -} - -func (m *MockMailbox) Purge() error { - args := m.Called() - return args.Error(0) -} - -func (m *MockMailbox) NewMessage() (datastore.Message, error) { - args := m.Called() - return args.Get(0).(datastore.Message), args.Error(1) -} - -func (m *MockMailbox) Name() string { - args := m.Called() - return args.String(0) -} - -func (m *MockMailbox) String() string { - args := m.Called() - return args.String(0) -} - -// Mock Message object -type MockMessage struct { - mock.Mock -} - -func (m *MockMessage) ID() string { - args := m.Called() - return args.String(0) -} - -func (m *MockMessage) From() string { - args := m.Called() - return args.String(0) -} - -func (m *MockMessage) To() []string { - args := m.Called() - return args.Get(0).([]string) -} - -func (m *MockMessage) Date() time.Time { - args := m.Called() - return args.Get(0).(time.Time) -} - -func (m *MockMessage) Subject() string { - args := m.Called() - return args.String(0) -} - -func (m *MockMessage) ReadHeader() (msg *mail.Message, err error) { - args := m.Called() - return args.Get(0).(*mail.Message), args.Error(1) -} - -func (m *MockMessage) ReadBody() (body *enmime.Envelope, err error) { - args := m.Called() - return args.Get(0).(*enmime.Envelope), args.Error(1) -} - -func (m *MockMessage) ReadRaw() (raw *string, err error) { - args := m.Called() - return args.Get(0).(*string), args.Error(1) -} - -func (m *MockMessage) RawReader() (reader io.ReadCloser, err error) { - args := m.Called() - return args.Get(0).(io.ReadCloser), args.Error(1) -} - -func (m *MockMessage) Size() int64 { - args := m.Called() - return int64(args.Int(0)) -} - -func (m *MockMessage) Append(data []byte) error { - // []byte arg seems to mess up testify/mock - return nil -} - -func (m *MockMessage) Close() error { - args := m.Called() - return args.Error(0) -} - -func (m *MockMessage) Delete() error { - args := m.Called() - return args.Error(0) -} - -func (m *MockMessage) String() string { - args := m.Called() - return args.String(0) -}