mirror of
https://github.com/jhillyerd/inbucket.git
synced 2025-12-18 01:57:02 +00:00
Move retention scanner into datastore pkg for #67
This commit is contained in:
James Hillyerd
182
datastore/retention.go
Normal file
182
datastore/retention.go
Normal file
@@ -0,0 +1,182 @@
|
||||
package datastore
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"expvar"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jhillyerd/inbucket/config"
|
||||
"github.com/jhillyerd/inbucket/log"
|
||||
)
|
||||
|
||||
var (
|
||||
retentionScanCompleted = time.Now()
|
||||
retentionScanCompletedMu sync.RWMutex
|
||||
|
||||
// History counters
|
||||
expRetentionDeletesTotal = new(expvar.Int)
|
||||
expRetentionPeriod = new(expvar.Int)
|
||||
expRetainedCurrent = new(expvar.Int)
|
||||
|
||||
// History of certain stats
|
||||
retentionDeletesHist = list.New()
|
||||
retainedHist = list.New()
|
||||
|
||||
// History rendered as comma delimited string
|
||||
expRetentionDeletesHist = new(expvar.String)
|
||||
expRetainedHist = new(expvar.String)
|
||||
)
|
||||
|
||||
func init() {
|
||||
rm := expvar.NewMap("retention")
|
||||
rm.Set("SecondsSinceScanCompleted", expvar.Func(secondsSinceRetentionScanCompleted))
|
||||
rm.Set("DeletesHist", expRetentionDeletesHist)
|
||||
rm.Set("DeletesTotal", expRetentionDeletesTotal)
|
||||
rm.Set("Period", expRetentionPeriod)
|
||||
rm.Set("RetainedHist", expRetainedHist)
|
||||
rm.Set("RetainedCurrent", expRetainedCurrent)
|
||||
|
||||
log.AddTickerFunc(func() {
|
||||
expRetentionDeletesHist.Set(log.PushMetric(retentionDeletesHist, expRetentionDeletesTotal))
|
||||
expRetainedHist.Set(log.PushMetric(retainedHist, expRetainedCurrent))
|
||||
})
|
||||
}
|
||||
|
||||
// RetentionScanner looks for messages older than the configured retention period and deletes them.
|
||||
type RetentionScanner struct {
|
||||
globalShutdown chan bool // Closes when Inbucket needs to shut down
|
||||
retentionShutdown chan bool // Closed after the scanner has shut down
|
||||
ds DataStore
|
||||
retentionPeriod time.Duration
|
||||
retentionSleep time.Duration
|
||||
}
|
||||
|
||||
// NewRetentionScanner launches a go-routine that scans for expired
|
||||
// messages, following the configured interval
|
||||
func NewRetentionScanner(ds DataStore, shutdownChannel chan bool) *RetentionScanner {
|
||||
cfg := config.GetDataStoreConfig()
|
||||
rs := &RetentionScanner{
|
||||
globalShutdown: shutdownChannel,
|
||||
retentionShutdown: make(chan bool),
|
||||
ds: ds,
|
||||
retentionPeriod: time.Duration(cfg.RetentionMinutes) * time.Minute,
|
||||
retentionSleep: time.Duration(cfg.RetentionSleep) * time.Millisecond,
|
||||
}
|
||||
// expRetentionPeriod is displayed on the status page
|
||||
expRetentionPeriod.Set(int64(cfg.RetentionMinutes * 60))
|
||||
return rs
|
||||
}
|
||||
|
||||
// Start up the retention scanner if retention period > 0
|
||||
func (rs *RetentionScanner) Start() {
|
||||
if rs.retentionPeriod <= 0 {
|
||||
log.Infof("Retention scanner disabled")
|
||||
close(rs.retentionShutdown)
|
||||
return
|
||||
}
|
||||
log.Infof("Retention configured for %v", rs.retentionPeriod)
|
||||
go rs.run()
|
||||
}
|
||||
|
||||
// run loops to kick off the scanner on the correct schedule
|
||||
func (rs *RetentionScanner) run() {
|
||||
start := time.Now()
|
||||
retentionLoop:
|
||||
for {
|
||||
// Prevent scanner from starting more than once a minute
|
||||
since := time.Since(start)
|
||||
if since < time.Minute {
|
||||
dur := time.Minute - since
|
||||
log.Tracef("Retention scanner sleeping for %v", dur)
|
||||
select {
|
||||
case _ = <-rs.globalShutdown:
|
||||
break retentionLoop
|
||||
case _ = <-time.After(dur):
|
||||
}
|
||||
}
|
||||
// Kickoff scan
|
||||
start = time.Now()
|
||||
if err := rs.doScan(); err != nil {
|
||||
log.Errorf("Error during retention scan: %v", err)
|
||||
}
|
||||
// Check for global shutdown
|
||||
select {
|
||||
case _ = <-rs.globalShutdown:
|
||||
break retentionLoop
|
||||
default:
|
||||
}
|
||||
}
|
||||
log.Tracef("Retention scanner shut down")
|
||||
close(rs.retentionShutdown)
|
||||
}
|
||||
|
||||
// doScan does a single pass of all mailboxes looking for messages that can be purged
|
||||
func (rs *RetentionScanner) doScan() error {
|
||||
log.Tracef("Starting retention scan")
|
||||
cutoff := time.Now().Add(-1 * rs.retentionPeriod)
|
||||
mboxes, err := rs.ds.AllMailboxes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
retained := 0
|
||||
// Loop over all mailboxes
|
||||
for _, mb := range mboxes {
|
||||
messages, err := mb.GetMessages()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Loop over all messages in mailbox
|
||||
for _, msg := range messages {
|
||||
if msg.Date().Before(cutoff) {
|
||||
log.Tracef("Purging expired message %v", msg.ID())
|
||||
err = msg.Delete()
|
||||
if err != nil {
|
||||
// Log but don't abort
|
||||
log.Errorf("Failed to purge message %v: %v", msg.ID(), err)
|
||||
} else {
|
||||
expRetentionDeletesTotal.Add(1)
|
||||
}
|
||||
} else {
|
||||
retained++
|
||||
}
|
||||
}
|
||||
// Sleep after completing a mailbox
|
||||
select {
|
||||
case <-rs.globalShutdown:
|
||||
log.Tracef("Retention scan aborted due to shutdown")
|
||||
return nil
|
||||
case <-time.After(rs.retentionSleep):
|
||||
// Reduce disk thrashing
|
||||
}
|
||||
}
|
||||
// Update metrics
|
||||
setRetentionScanCompleted(time.Now())
|
||||
expRetainedCurrent.Set(int64(retained))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Join does not retun until the retention scanner has shut down
|
||||
func (rs *RetentionScanner) Join() {
|
||||
if rs.retentionShutdown != nil {
|
||||
select {
|
||||
case <-rs.retentionShutdown:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func setRetentionScanCompleted(t time.Time) {
|
||||
retentionScanCompletedMu.Lock()
|
||||
defer retentionScanCompletedMu.Unlock()
|
||||
retentionScanCompleted = t
|
||||
}
|
||||
|
||||
func getRetentionScanCompleted() time.Time {
|
||||
retentionScanCompletedMu.RLock()
|
||||
defer retentionScanCompletedMu.RUnlock()
|
||||
return retentionScanCompleted
|
||||
}
|
||||
|
||||
func secondsSinceRetentionScanCompleted() interface{} {
|
||||
return time.Since(getRetentionScanCompleted()) / time.Second
|
||||
}
|
||||
67
datastore/retention_test.go
Normal file
67
datastore/retention_test.go
Normal file
@@ -0,0 +1,67 @@
|
||||
package datastore
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDoRetentionScan(t *testing.T) {
|
||||
// Create mock objects
|
||||
mds := &MockDataStore{}
|
||||
|
||||
mb1 := &MockMailbox{}
|
||||
mb2 := &MockMailbox{}
|
||||
mb3 := &MockMailbox{}
|
||||
|
||||
// Mockup some different aged messages (num is in hours)
|
||||
new1 := mockMessage(0)
|
||||
new2 := mockMessage(1)
|
||||
new3 := mockMessage(2)
|
||||
old1 := mockMessage(4)
|
||||
old2 := mockMessage(12)
|
||||
old3 := mockMessage(24)
|
||||
|
||||
// First it should ask for all mailboxes
|
||||
mds.On("AllMailboxes").Return([]Mailbox{mb1, mb2, mb3}, nil)
|
||||
|
||||
// Then for all messages on each box
|
||||
mb1.On("GetMessages").Return([]Message{new1, old1, old2}, nil)
|
||||
mb2.On("GetMessages").Return([]Message{old3, new2}, nil)
|
||||
mb3.On("GetMessages").Return([]Message{new3}, nil)
|
||||
|
||||
// Test 4 hour retention
|
||||
rs := &RetentionScanner{
|
||||
ds: mds,
|
||||
retentionPeriod: 4*time.Hour - time.Minute,
|
||||
retentionSleep: 0,
|
||||
}
|
||||
if err := rs.doScan(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
// Check our assertions
|
||||
mds.AssertExpectations(t)
|
||||
mb1.AssertExpectations(t)
|
||||
mb2.AssertExpectations(t)
|
||||
mb3.AssertExpectations(t)
|
||||
|
||||
// Delete should not have been called on new messages
|
||||
new1.AssertNotCalled(t, "Delete")
|
||||
new2.AssertNotCalled(t, "Delete")
|
||||
new3.AssertNotCalled(t, "Delete")
|
||||
|
||||
// Delete should have been called once on old messages
|
||||
old1.AssertNumberOfCalls(t, "Delete", 1)
|
||||
old2.AssertNumberOfCalls(t, "Delete", 1)
|
||||
old3.AssertNumberOfCalls(t, "Delete", 1)
|
||||
}
|
||||
|
||||
// Make a MockMessage of a specific age
|
||||
func mockMessage(ageHours int) *MockMessage {
|
||||
msg := &MockMessage{}
|
||||
msg.On("ID").Return(fmt.Sprintf("MSG[age=%vh]", ageHours))
|
||||
msg.On("Date").Return(time.Now().Add(time.Duration(ageHours*-1) * time.Hour))
|
||||
msg.On("Delete").Return(nil)
|
||||
return msg
|
||||
}
|
||||
Reference in New Issue
Block a user